def test_s3_api_bucket_add_user_admin(mock_set_up_bucket_on_aws, client):
mock_set_up_bucket_on_aws.return_value = (True, '')
user = factories.User(admin=True)
bucket1 = factories.S3Bucket()
bucket1.users.append(user)
bucket2 = factories.S3Bucket()
bucket2.users.append(user)
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.post(
'/s3/api/bucket',
headers={'Content-Type': 'application/json'},
data=json.dumps({
'access-key-id': 'id',
'secret-access-key': 'key',
'bucket-name': 'name',
})
)
assert response.status_code == 200
assert get_response_data(response) == {
'bucket': {
'id': mock.ANY,
'name': 'name',
'active': True,
'integration_name': 's3'
},
'error': None,
}
def test_s3_api_bucket_add_user_on_free_plan(client):
user = factories.User()
bucket1 = factories.S3Bucket()
bucket1.users.append(user)
bucket2 = factories.S3Bucket()
bucket2.users.append(user)
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.post(
'/s3/api/bucket',
headers={'Content-Type': 'application/json'},
data=json.dumps({
'access-key-id': 'id',
'secret-access-key': 'key',
'bucket-name': 'name',
})
)
assert response.status_code == 200
assert get_response_data(response) == {
'bucket': None,
'error': 'Free plan users can only have {} active buckets'.format(
settings.MAX_S3_BUCKETS_ON_FREE_PLAN)
}
def test_get_user_buckets_count():
user = factories.User()
factories.S3Bucket(name='test-bucket-1', users=[user])
factories.S3Bucket(name='test-bucket-2', users=[user])
buckets_count = get_user_buckets_count(user.id)
assert buckets_count == 2
def test_get_user_buckets():
user = factories.User()
factories.S3Bucket(name='test-bucket-1', users=[user])
factories.S3Bucket(name='test-bucket-2')
buckets = get_user_buckets(user.id)
assert len(buckets) == 1
assert buckets[0].name == 'test-bucket-1'
def test_create_bucket_already_exists_and_not_active():
bucket = factories.S3Bucket(name='test-bucket-2', active=False)
updated_bucket = create_bucket('test-bucket-2')
assert updated_bucket.id == bucket.id
assert updated_bucket.active is True
def test_s3_api_bucket_list(client):
user = factories.User()
bucket1 = factories.S3Bucket(name='name1', users=[user])
bucket2 = factories.S3Bucket(name='name2', users=[user])
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.get('/s3/api/bucket')
assert response.status_code == 200
assert get_response_data(response) == {
'buckets': [
{'id': bucket1.id, 'name': bucket1.name, 'active': bucket1.active,
'integration_name': 's3'},
{'id': bucket2.id, 'name': bucket2.name, 'active': bucket2.active,
'integration_name': 's3'},
],
'error': None,
}
def test_s3_api_bucket_deactivate_no_access_to_bucket(client):
user = factories.User()
bucket = factories.S3Bucket(
conf={'access_key_id': 'test', 'secret_access_key': 'test'})
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.get('/s3/api/bucket/{}/deactivate'.format(bucket.id))
assert response.status_code == 403
def test_badge_source_exists_but_no_jobs(client):
bucket = factories.S3Bucket()
response = client.get('/badge/s3/{}.svg'.format(bucket.name))
assert response.content_type == 'image/svg+xml'
assert b'>unknown</text></g>' in response.get_data()
def test_badge_source_exists_job_failure(client):
bucket = factories.S3Bucket()
factories.Job(source=bucket, integration_name='s3', status='failure')
response = client.get('/badge/s3/{}.svg'.format(bucket.name))
assert response.content_type == 'image/svg+xml'
assert b'>invalid</text></g>' in response.get_data()
def test_delete_bucket():
user = factories.User()
factories.GithubRepo(users=[user], active=True)
bucket = factories.S3Bucket(users=[user], active=True)
bucket = database['session'].query(S3Bucket).one()
database['session'].delete(bucket)
database['session'].commit()
users = database['session'].query(User).all()
buckets = database['session'].query(S3Bucket).all()
assert len(users) == 1
assert len(users[0].sources) == 1
assert len(buckets) == 0
def test_access_key_id_del():
bucket = factories.S3Bucket(access_key_id='xxx')
del bucket.access_key_id
assert bucket.access_key_id is None
database['session'].commit()
bucket_db = database['session'].query(S3Bucket).first()
assert bucket_db.access_key_id is None
def test_secret_access_key_del():
bucket = factories.S3Bucket(secret_access_key='xxx')
del bucket.secret_access_key
assert bucket.secret_access_key is None
database['session'].commit()
bucket_db = database['session'].query(S3Bucket).first()
assert bucket_db.secret_access_key is None
def test_deactivate_bucket():
factories.S3Bucket(name='test-bucket-2', active=True)
deactivate_bucket('test-bucket-2')
buckets = database['session'].query(S3Bucket).all()
assert len(buckets) == 1
assert buckets[0].name == 'test-bucket-2'
assert buckets[0].active is False
def test_secret_access_key_set():
bucket = factories.S3Bucket()
bucket.secret_access_key = 'xxx'
# Encrypted
assert bucket.conf['secret_access_key'].startswith('gAAA')
database['session'].commit()
bucket_db = database['session'].query(S3Bucket).first()
assert bucket_db.secret_access_key == 'xxx'
def test_s3_api_bucket_deactivate_failure(mock_disable_bucket_on_aws, client):
user = factories.User()
bucket = factories.S3Bucket(
users=[user], active=True,
conf={'access_key_id': 'test', 'secret_access_key': 'test'})
mock_disable_bucket_on_aws.return_value = (False, 'Some error happened')
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.get('/s3/api/bucket/{}/deactivate'.format(bucket.id))
assert response.status_code == 200
assert get_response_data(response) == {
'error': 'Error removing bucket integration. Some error happened',
}
def test_s3_api_bucket_delete(mock_disable_bucket_on_aws, client):
user = factories.User()
bucket = factories.S3Bucket(
users=[user], active=True,
conf={'access_key_id': 'test', 'secret_access_key': 'test'})
mock_disable_bucket_on_aws.return_value = (True, '')
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.delete('/s3/api/bucket/{}'.format(bucket.id))
buckets = database['session'].query(S3Bucket).all()
assert len(buckets) == 0
assert response.status_code == 200
assert get_response_data(response) == {
'error': None,
}
def test_s3_api_bucket(client):
user = factories.User()
bucket = factories.S3Bucket(name='name', users=[user])
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.get('/s3/api/bucket/%s' % bucket.id)
assert response.status_code == 200
assert get_response_data(response) == {
'bucket': {
'id': bucket.id,
'name': bucket.name,
'active': bucket.active,
},
'error': None,
}
def test_s3_api_bucket_activate(mock_set_up_bucket_on_aws, client):
user = factories.User()
bucket = factories.S3Bucket(
users=[user], active=False,
conf={'access_key_id': 'test', 'secret_access_key': 'test'})
mock_set_up_bucket_on_aws.return_value = (True, '')
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.get('/s3/api/bucket/{}/activate'.format(bucket.id))
buckets = database['session'].query(S3Bucket).all()
assert buckets[0].name == bucket.name
assert buckets[0].active is True
assert response.status_code == 200
assert get_response_data(response) == {
'error': None,
}
def test_s3_api_bucket_add_already_exists(client):
user = factories.User()
bucket = factories.S3Bucket()
with client.session_transaction() as session:
session['user_id'] = user.id
response = client.post('/s3/api/bucket',
headers={'Content-Type': 'application/json'},
data=json.dumps({
'access-key-id': 'test',
'secret-access-key': 'test',
'bucket-name': bucket.name
}))
assert response.status_code == 200
assert get_response_data(response) == {
'bucket': None,
'error': 'Bucket already exists',
}
def test_badge_source_exists_picks_last_finished_job(client):
bucket = factories.S3Bucket()
factories.Job(source=bucket,
integration_name='s3',
status='success',
finished=datetime.datetime.utcnow())
factories.Job(source=bucket,
integration_name='s3',
status='failure',
finished=datetime.datetime.utcnow())
factories.Job(source=bucket, integration_name='s3', status='running')
response = client.get('/badge/s3/{}.svg'.format(bucket.name))
assert response.content_type == 'image/svg+xml'
assert b'>invalid</text></g>' in response.get_data()
def test_s3_hook_bucket_success(mock_1, client):
factories.S3Bucket(name='test-bucket')
data = {'Records': [{'s3': {'bucket': {'name': 'test-bucket'}}}]}
sig = create_signature(settings.S3_LAMBDA_HOOK_SECRET, json.dumps(data))
response = client.post(
'/s3/hook', data=json.dumps(data),
content_type='application/json',
headers={'X-GoodTables-Signature': sig})
assert response.status_code == 200
body = response.get_data(as_text=True)
job_id = json.loads(body)['job_id']
jobs = database['session'].query(Job).all()
assert jobs[0].id == job_id
def test_get_validation_conf():
bucket = factories.S3Bucket(name='test=bucket',
conf={
'access_key_id': 'test',
'secret_access_key': 'test'
})
job = factories.Job(source=bucket)
def mock_make_api_call(self, operation_name, kwarg):
if operation_name == 'GetObject':
return mock_responses.s3_get_object
elif operation_name == 'ListObjects':
return mock_responses.s3_list_objects
else:
assert 0
def mock_generate_presigned_url(*args, **kwargs):
return 'https://example.com/presigned/{}'.format(
kwargs['Params']['Key'])
with mock.patch('botocore.client.BaseClient._make_api_call',
new=mock_make_api_call), \
mock.patch('botocore.signers.generate_presigned_url',
new=mock_generate_presigned_url):
expected = {
'source': [{
'source':
'https://example.com/presigned/councillors-address-3.csv'
}, {
'source':
'https://example.com/presigned/councillors-address2.csv'
}],
'settings': {
'error_limit': 1
}
}
assert get_validation_conf('test-gtio-1', job.id) == expected
def test_access_key_id_get():
bucket = factories.S3Bucket(access_key_id='xxx')
assert bucket.access_key_id == 'xxx'
def test_secret_access_key_get_unencrypted():
bucket = factories.S3Bucket(conf={'secret_access_key': 'xxx'})
assert bucket.secret_access_key is None
def test_secret_access_key_get():
bucket = factories.S3Bucket(secret_access_key='xxx')
assert bucket.secret_access_key == 'xxx'
def test_access_key_id_get_unencrypted():
bucket = factories.S3Bucket(conf={'access_key_id': 'xxx'})
assert bucket.access_key_id is None
def test_create_bucket_already_exists_and_active():
factories.S3Bucket(name='test-bucket-2', active=True)
assert create_bucket('test-bucket-2') is False
