def test_api_job_list(client):
job1 = factories.Job()
job2 = factories.Job()
response = client.get('/api/job')
assert _data(response) == [job2.id, job1.id]
def test_create_job_are_different():
job1 = factories.Job()
job2 = factories.Job()
assert job1.id != job2.id
assert job1.created != job2.created
def test_api_source_job_list_foreign_private_source(get):
source = factories.Source(integration_name='api', conf={'private': True})
job1 = factories.Job(source=source)
job2 = factories.Job(source=source)
user = factories.User()
token = user.create_api_token()
code, data = get('/api/source/%s/job' % source.id, token=token.token)
assert code == 403
assert data['message'] == 'Forbidden'
def test_api_source_job_list(get):
source = factories.Source(integration_name='api')
job1 = factories.Job(source=source)
job2 = factories.Job(source=source)
user = factories.User()
token = user.create_api_token()
code, data = get('/api/source/%s/job' % source.id, token=token.token)
assert code == 200
assert data['jobs'][0]['id'] == job2.id
assert data['jobs'][1]['id'] == job1.id
def test_find():
job1 = factories.Job()
job2 = factories.Job()
job3 = factories.Job()
jobs = models.job.find()
assert len(jobs) == 3
assert jobs == [job3.to_dict(), job2.to_dict(), job1.to_dict()]
def test_find_offset():
job1 = factories.Job()
job2 = factories.Job()
factories.Job()
jobs = models.job.find(offset=1)
assert len(jobs) == 2
assert jobs == [job2.to_dict(), job1.to_dict()]
def test_find_limit():
factories.Job()
job2 = factories.Job()
job3 = factories.Job()
jobs = models.job.find(limit=2)
assert len(jobs) == 2
assert jobs == [job3.to_dict(), job2.to_dict()]
def test_find_filter():
job1 = factories.Job(integration_name='github')
factories.Job()
jobs = models.job.find(
filters=[models.job.Job.integration_name == 'github'])
assert len(jobs) == 1
assert jobs[0]['id'] == job1.id
assert jobs[0]['integration_name'] == 'github'
def test_s3_home(client):
job1 = factories.Job(integration_name='s3')
job2 = factories.Job(integration_name='github')
response = client.get('/s3/')
body = response.get_data(as_text=True)
# TODO: improve when final UI is in place
assert 'S3' in body
assert job1.id in body
assert job2.id not in body
def test_signal_sets_status(set_commit_status):
conf = {
'owner': 'test-org',
'repo': 'test-repo',
'sha': 'abcde',
'is_pr': True
}
integration = factories.Integration()
repo = factories.GithubRepo(integration_name=integration.name)
job = factories.Job(integration_name='github',
conf=conf,
status='success',
number=3,
source=repo)
post_task_handler(retval=job.to_dict(),
kwargs={'job_id': job.id},
state='SUCCESS')
set_commit_status.assert_called_with(
'success',
owner='test-org',
repo='test-repo',
sha='abcde',
is_pr=True,
job_number=3,
tokens=job.source.tokens,
)
def test_validate(_inspect):
# We need to save it on the DB so the session used by the tasks can find it
job = factories.Job(_save_in_db=True)
mock_report = {
'valid': True,
'tables': [],
'errors': [],
'warnings': [],
'table-count': 1,
}
_inspect.return_value = mock_report
validation_conf = {
'source': [{
'source': 'url1'
}, {
'source': 'url2'
}],
'settings': {}
}
validate(validation_conf, job_id=job.id)
_inspect.assert_called_with(validation_conf['source'], preset='nested')
jobs = models.job.find()
assert len(jobs) == 1
updated_job = jobs[0]
assert updated_job['id'] == job.id
assert updated_job['report'] == mock_report
assert isinstance(updated_job['finished'], datetime.datetime)
def test_update_job_stored_in_db():
job = factories.Job()
report = {'nice': 'csv'}
finished = datetime.datetime.utcnow()
params = {
'id': job.id,
'status': 'success',
'report': report,
'finished': finished,
}
models.job.update(params)
# Make sure that we are not checking the cached object in the session
database['session'].remove()
updated_job = database['session'].query(models.job.Job).get(job.id)
assert updated_job
assert updated_job.id == job.id
assert updated_job.status == 'success'
assert updated_job.integration_name == 'api'
assert updated_job.conf is None
assert updated_job.created
assert updated_job.finished.replace(tzinfo=None) == finished
assert updated_job.report == report
assert updated_job.error is None
def test_different_integration(_update):
job = factories.Job(integration_name='github', status='success')
post_task_handler(retval=job.to_dict(), state='SUCCESS')
_update.assert_not_called()
def test_badge_source_exists_picks_last_finished_job(client):
bucket = factories.S3Bucket()
factories.Job(source=bucket,
integration_name='s3',
status='success',
finished=datetime.datetime.utcnow())
factories.Job(source=bucket,
integration_name='s3',
status='failure',
finished=datetime.datetime.utcnow())
factories.Job(source=bucket, integration_name='s3', status='running')
response = client.get('/badge/s3/{}.svg'.format(bucket.name))
assert response.content_type == 'image/svg+xml'
assert b'>invalid</text></g>' in response.get_data()
def test_find_default_limit():
for i in range(0, 15):
factories.Job()
jobs = models.job.find()
assert len(jobs) == 10
def test_api_source_job_get(get):
user = factories.User()
token = user.create_api_token()
source = factories.Source(users=[user],
integration_name='api', conf={'private': True})
job = factories.Job(source=source)
code, data = get('/api/source/%s/job/%s' % (source.id, job.id), token=token.token)
assert code == 200
assert data['job']['id'] == job.id
def test_badge_source_exists_job_failure(client):
bucket = factories.S3Bucket()
factories.Job(source=bucket, integration_name='s3', status='failure')
response = client.get('/badge/s3/{}.svg'.format(bucket.name))
assert response.content_type == 'image/svg+xml'
assert b'>invalid</text></g>' in response.get_data()
def test_create_job():
job = factories.Job()
assert job.status == 'created'
assert job.integration_name == 'api'
job_db = database['session'].query(Job).get(job.id)
assert job_db.id == job.id
def test_signal_no_github(set_commit_status):
job = factories.Job(integration_name='s3')
post_task_handler(retval=job.to_dict(),
kwargs={'job_id': job.id},
state='SUCCESS')
assert not set_commit_status.called
def test_signal_exception(_update):
job = factories.Job(integration_name='s3', status='success')
post_task_handler(retval=Exception(),
kwargs={'job_id': job.id},
state='SUCCESS')
_update.assert_not_called()
def test_site_get_job(client):
job = factories.Job()
response = client.get('/jobs/{0}'.format(job.id))
# TODO: Test actual content when implemented
body = response.get_data(as_text=True)
assert 'html' in body
def test_badge_works_on_github_repos(client):
repo = factories.GithubRepo()
factories.Job(source=repo, integration_name='github', status='success')
response = client.get('/badge/github/{}/{}.svg'.format(
repo.owner, repo.repo))
assert response.content_type == 'image/svg+xml'
assert b'>valid</text></g>' in response.get_data()
def test_set_job_number(celery_app):
source1 = factories.Source(integration_name='github', _save_in_db=True)
source2 = factories.Source(integration_name='github', _save_in_db=True)
assert source1.job_number == 1
assert source2.job_number == 1
job1 = factories.Job(source=source1, _save_in_db=True)
job2 = factories.Job(source=source1, _save_in_db=True)
job3 = factories.Job(source=source2, _save_in_db=True)
job4 = factories.Job(source=source1, _save_in_db=True)
assert source1.job_number == 4
assert source2.job_number == 2
assert job1.number == 1
assert job2.number == 2
assert job3.number == 1
assert job4.number == 3
def test_validate_skip_rows():
source = 'text://a,b\n1,2\n#comment'
format = 'csv'
# Without skip rows
job = factories.Job()
conf = {'source': [{'source': source, 'format': format}]}
job = validate(conf, job_id=job.id)
assert job['report']['valid'] is False
# With skip rows
job = factories.Job()
conf = {
'source': [{
'source': source,
'format': format,
'skip_rows': ['#']
}]
}
job = validate(conf, job_id=job.id)
assert job['report']['valid'] is True
def test_validate_more_than_10_tables_no_settings():
# We need to save it on the DB so the session used by the tasks can find it
job = factories.Job()
validation_conf = {
'source': [{
'source': 'http://example.com/file{}'.format(i)
} for i in range(12)],
}
validate(validation_conf, job_id=job.id)
jobs = models.job.find()
assert len(jobs[0]['report']['tables']) == 12
def test_json_fields_mutable():
job = factories.Job(id='my-id', conf={'a': '1'}, _save_in_db=True)
job.conf.update({'b': '2'})
database['session'].add(job)
database['session'].commit()
database['session'].remove()
job = database['session'].query(Job).get('my-id')
assert job.conf == {'a': '1', 'b': '2'}
def test_api_get_job(client):
job = factories.Job()
response = client.get('/api/job/{0}'.format(job.id))
data = _data(response)
# TODO: Update after #19
assert 'report' in data
assert data['id'] == job.id
assert 'created' in data
assert 'status' in data
def test_validate_more_than_100_tables_even_if_settings():
# We need to save it on the DB so the session used by the tasks can find it
job = factories.Job()
validation_conf = {
'source': [{
'source': 'http://example.com/file{}'.format(i)
} for i in range(110)],
'settings': {
'table_limit': 200
}
}
validate(validation_conf, job_id=job.id)
jobs = models.job.find()
assert len(jobs[0]['report']['tables']) == settings.MAX_TABLES_PER_SOURCE
def test_get_job_outputs_dict():
# Actually save it to the DB so we can test retrieving it
job_db = factories.Job(_save_in_db=True).to_dict()
database['session'].remove()
job = models.job.get(job_db['id'])
assert job['id'] == job_db['id']
assert job['status'] == job_db['status']
assert job['integration_name'] == job_db['integration_name']
assert job['conf'] == job_db['conf']
assert job['created'] == job_db['created']
assert job['finished'] == job_db['finished']
assert job['report'] == job_db['report']
assert job['error'] == job_db['error']
def test_validate_datapackage_file(sample_datapackage):
job = factories.Job(_save_in_db=True)
validation_conf = {
'source': [
{
'preset': 'datapackage',
'source': 'file0'
},
],
'settings': {},
}
files = {
'file0': sample_datapackage.name,
}
job = validate(validation_conf, job.id, files)
assert job['error'] is None
