def setUp(self):
ef.clear()
self.app = Client()
self.account = ef.account(type=AccountType.ADMIN.value)
self.ci = ef.catalogue_item(spec=[
{
'name': 'product',
'type': 'STRING',
'is_nullable': True,
'is_enum': True,
'size': None,
'distribution': None,
},
{
'name': 'price',
'type': 'INTEGER',
'is_nullable': False,
'is_enum': False,
'size': None,
'distribution': None,
},
],
executor_type='ATHENA')
token = AuthToken.encode(self.account)
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
class AuthTokenCommandsTestCase(TestCase):
uri = reverse('account:auth.auth_token')
@pytest.fixture(autouse=True)
def initfixtures(self, mocker):
self.mocker = mocker
def setUp(self):
ef.clear()
self.app = Client()
def test_post(self):
self.mocker.patch.object(
AuthToken,
'encode'
).return_value = 'fd78cd7d87f'
r = ef.auth_request()
r.account = ef.account()
r.save()
response = self.app.post(
self.uri,
data=json.dumps({
'request_uuid': str(r.uuid),
}),
content_type='application/json')
assert response.status_code == 201
assert response.json() == {
'@event': 'AUTH_TOKEN_CREATED',
'@type': 'auth_token',
'token': 'fd78cd7d87f',
}
@override_settings(AUTH_REQUEST_EXPIRATION_DELTA=timedelta(seconds=120))
@freeze_time('2014-10-10 12:00:00')
def test_post_expired(self):
r = ef.auth_request()
r.account = ef.account()
r.created_datetime = timezone.make_aware(
datetime(2014, 10, 10, 11, 57, 0))
r.save()
response = self.app.post(
self.uri,
data=json.dumps({
'request_uuid': str(r.uuid),
}),
content_type='application/json')
assert response.status_code == 400
assert response.json() == {
'@event': 'EXPIRED_AUTH_REQUEST_DETECTED',
'@type': 'error',
}
def setUp(self):
self.app = Client()
self.user_id = 56
self.auth_headers = {
'HTTP_X_ACCOUNT_TYPE': 'PREMIUM',
'HTTP_X_USER_ID': self.user_id,
}
def test_multiple_different_responses_single_endpoint(self):
examples_file = self.prepare_example_file()
Client().get('/test/it/?type=a')
Client().get('/test/it/?type=b')
Client().get('/test/it/?type=c')
Client().get('/test/it/?type=d')
assert json.loads(examples_file.read()) == {
'GET_IT': {
'200 (LISTED)': {
'method': 'get',
'description': 'LISTED',
'request': {
'path': '/test/it/?type=b',
},
'response': {
'status': 200,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'LISTED',
'hello': 'get.b',
},
}
},
'200 (LISTED_AGAIN)': {
'method': 'get',
'description': 'LISTED_AGAIN',
'request': {
'path': '/test/it/?type=c',
},
'response': {
'status': 200,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'LISTED_AGAIN',
'hello': 'get.c',
},
},
},
'404 (ERROR_LISTED)': {
'method': 'get',
'description': 'ERROR_LISTED',
'request': {
'path': '/test/it/?type=d',
},
'response': {
'status': 404,
'content_type': 'application/json',
'content': {
'@type': 'error',
'@event': 'ERROR_LISTED',
'hello': 'get.d',
},
},
},
},
}
def test_body_examples_for_post_and_put(self):
examples_file = self.prepare_example_file()
Client().post('/test/it/',
data=json.dumps({'create': 'it'}),
content_type='application/json')
Client().put('/test/it/', json={'update': 'it', 'please': 'now'})
assert json.loads(examples_file.read()) == {
'POST_IT': {
'201 (CREATED)': {
'method': 'post',
'description': 'CREATED',
'response': {
'status': 201,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'CREATED',
'hello': 'post',
}
},
'request': {
'path': '/test/it/',
'headers': {
'CONTENT-TYPE': 'application/json',
},
'content': {
'create': 'it',
}
}
},
},
'PUT_IT': {
'200 (UPDATED)': {
'method': 'put',
'description': 'UPDATED',
'response': {
'status': 200,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'UPDATED',
'hello': 'put',
},
},
'request': {
'path': '/test/it/',
'headers': {
'CONTENT-TYPE': 'application/json',
},
'content': {
'update': 'it',
'please': 'now',
}
}
}
},
}
def setUp(self):
ef.clear()
self.app = Client()
self.account = ef.account(type=AccountType.ADMIN.value)
token = AuthToken.encode(self.account)
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
class CatalogueItemSampleAndDistributionsCommandsTestCase(TestCase):
def get_uri(self, item_id):
return reverse('catalogue:items.element.samples_and_distributions',
kwargs={'item_id': item_id})
@pytest.fixture(autouse=True)
def initfixtures(self, mocker):
self.mocker = mocker
def setUp(self):
ef.clear()
self.app = Client()
self.account = ef.account(type=AccountType.ADMIN.value)
token = AuthToken.encode(self.account)
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
#
# UPDATE
#
def test_put_200(self):
update_samples_and_distributions = self.mocker.patch.object(
CatalogueItem, 'update_samples_and_distributions')
a_0 = ef.account()
a_1 = ef.account()
ci = ef.catalogue_item(name='temperatures',
created_by=a_0,
updated_by=a_1)
response = self.app.put(self.get_uri(ci.id), **self.headers)
assert response.status_code == 200
ci.refresh_from_db()
assert response.json() == {
'@event': 'CATALOGUEITEM_WITH_SAMPLE_AND_DISTRIBUTION_UPDATED',
'@type': 'empty',
}
assert update_samples_and_distributions.call_count == 1
def test_put_404(self):
response = self.app.put(self.get_uri(9022), **self.headers)
assert response.status_code == 404
assert response.json() == {
'@event': 'COULD_NOT_FIND_CATALOGUEITEM',
'@type': 'error',
'@authorizer': {
'account_id': self.account.id,
},
}
def setUp(self):
self.app = Client()
self.auth_headers = {
'HTTP_X_CS_ACCOUNT_TYPE': 'ADMIN',
'HTTP_X_CS_USER_ID': 190,
}
self.lily_dir = self.tmpdir.mkdir('.lily')
self.commands_dir = self.lily_dir.mkdir('commands')
MockConfig._lily_path = str(self.lily_dir)
self.mocker.patch(
'entrypoint.commands.Config', MockConfig)
def test_create_file_if_does_not_exist(self):
examples_file_dir = self.tmpdir.mkdir('docs')
filepath = os.path.join(str(examples_file_dir), 'examples007.json')
get_examples_filepath = self.mocker.patch(
'lily.base.test.get_examples_filepath')
get_examples_filepath.return_value = filepath
response = Client().get('/test/it/')
examples_file = open(filepath, 'r')
assert response.status_code == 200
assert json.loads(examples_file.read()) == {
'GET_IT': {
'200 (LISTED)': {
'method': 'get',
'description': 'LISTED',
'request': {
'path': '/test/it/',
},
'response': {
'status': 200,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'LISTED',
'hello': 'get.a',
},
},
},
},
}
def test_with_extra_desc(self):
examples_file = self.prepare_example_file()
response = Client().get('/test/it/', extra_desc='special')
assert response.status_code == 200
assert json.loads(examples_file.read()) == {
'GET_IT': {
'200 (LISTED) - special': {
'method': 'get',
'description': 'LISTED',
'request': {
'path': '/test/it/',
},
'response': {
'status': 200,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'LISTED',
'hello': 'get.a',
},
},
},
},
}
class AuthRequestCommandsTestCase(TestCase):
uri = reverse('account:auth.requests')
def setUp(self):
ef.clear()
self.app = Client()
def test_post(self):
assert AuthRequest.objects.count() == 0
response = self.app.post(self.uri)
assert response.status_code == 201
assert AuthRequest.objects.count() == 1
r = AuthRequest.objects.all().first()
assert response.json() == {
'@event': 'AUTH_REQUEST_CREATED',
'@type': 'auth_request',
'authenticate_ui_uri': (
'/accounts/auth_requests/{}/authenticate/ui/'.format(
str(r.uuid))),
'request_uuid': str(r.uuid),
}
class DownloadRequestEstimateCommandsTestCase(TestCase):
uri = reverse('downloader:requests.estimate')
@pytest.fixture(autouse=True)
def initfixtures(self, mocker):
self.mocker = mocker
def setUp(self):
ef.clear()
self.app = Client()
self.account = ef.account(type=AccountType.ADMIN.value)
token = AuthToken.encode(self.account)
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
def test_post_200(self):
estimate_size = self.mocker.patch.object(DownloadRequest.objects,
'estimate_size')
estimate_size.return_value = 1234
ci = ef.catalogue_item(spec=[
{
'name': 'product',
'type': 'STRING',
'is_nullable': True,
'is_enum': True,
'size': None,
'distribution': None,
},
{
'name': 'price',
'type': 'INTEGER',
'is_nullable': False,
'is_enum': False,
'size': None,
'distribution': None,
},
])
response = self.app.post(self.uri,
data=json.dumps({
'spec': {
'columns': ['product', 'price'],
'filters': [],
'randomize_ratio': 0.9,
},
'catalogue_item_id': ci.id,
}),
content_type='application/json',
**self.headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'SIZE_OF_DOWNLOAD_REQUEST_ESTIMATED',
'@type': 'download_request_estimated_size',
'estimated_size': 1234,
}
def test_http_response(self):
examples_file = self.prepare_example_file()
response = Client().get('/http/it/')
assert response.status_code == 200
assert json.loads(examples_file.read()) == {
'HTTP_IT': {
'200': {
'method': 'get',
'description': None,
'request': {
'path': '/http/it/',
},
'response': {
'status': 200,
'content_type': 'text/html; charset=utf-8',
'content': 'http it!',
},
},
},
}
class AuthRequestAttachAccountCommandsTestCase(TestCase):
uri = reverse('account:auth.requests.attach_account')
@pytest.fixture(autouse=True)
def initfixtures(self, mocker):
self.mocker = mocker
def setUp(self):
ef.clear()
self.app = Client()
def test_post(self):
r = ef.auth_request()
self.mocker.patch(
'account.models.validate_token'
).return_value = '*****@*****.**'
assert Account.objects.count() == 0
response = self.app.post(
self.uri,
data=json.dumps({
'request_uuid': str(r.uuid),
'oauth_token': 'some-oauth-token',
'email': '*****@*****.**',
}),
content_type='application/json')
assert response.status_code == 200
assert response.json() == {
'@event': 'ACCOUNT_TO_AUTH_REQUEST_ATTACHED',
'@type': 'empty',
}
assert Account.objects.count() == 1
a = Account.objects.all().first()
assert a.email == '*****@*****.**'
def test_post__broken_body(self):
self.mocker.patch(
'account.models.validate_token'
).return_value = '*****@*****.**'
assert Account.objects.count() == 0
response = self.app.post(
self.uri,
data=json.dumps({
'request_uuid': 'some-uuid',
'email': '*****@*****.**',
}),
content_type='application/json')
assert response.status_code == 400
assert response.json() == {
'@event': 'BODY_DID_NOT_VALIDATE',
'@type': 'error',
'errors': {
'oauth_token': ['This field is required.'],
'request_uuid': ['Must be a valid UUID.'],
},
}
assert Account.objects.count() == 0
def test_post__request_does_not_exist(self):
assert Account.objects.count() == 0
response = self.app.post(
self.uri,
data=json.dumps({
'request_uuid': str(uuid1()),
'oauth_token': 'some-auth-token',
'email': '*****@*****.**',
}),
content_type='application/json')
assert response.status_code == 404
assert response.json() == {
'@event': 'COULD_NOT_FIND_AUTHREQUEST',
'@type': 'error',
}
assert Account.objects.count() == 0
class MySignCommandsTestCase(TestCase):
uri = reverse('test.sign')
@pytest.fixture(autouse=True)
def initfixtures(self, mocker):
self.mocker = mocker
def setUp(self):
self.app = Client()
self.user_id = 56
self.auth_headers = {
'HTTP_X_ACCOUNT_TYPE': 'PREMIUM',
'HTTP_X_USER_ID': self.user_id,
}
@override_settings(AWS_S3_ACCESS_KEY="access.key", AWS_S3_REGION="central")
def test_get_200(self):
sign_mock = self.mocker.patch.object(MySignCommands, 'sign')
sign_mock.side_effect = [
'signature 1',
'signature 2',
'signature 3',
'signature 4',
]
get_signature_mock = self.mocker.patch.object(MySignCommands,
'get_signature')
get_signature_mock.return_value = "af52522c5afb83b5348ed06b5fbd0c"
response = self.app.get(self.uri,
data={
"to_sign": "hi there",
"datetime": "20171201T123112Z",
},
**self.auth_headers)
assert response.status_code == 200
assert response.content == b"af52522c5afb83b5348ed06b5fbd0c"
assert (get_signature_mock.call_args_list == [
call('signature 4', 'hi there')
])
assert (sign_mock.call_args_list == [
call(b'AWS4access.key', '20171201'),
call('signature 1', 'central'),
call('signature 2', 's3'),
call('signature 3', 'aws4_request'),
])
def test_get_400__missing_fields(self):
# -- missing to sign
response = self.app.get(self.uri,
data={"datetime": "20171201T123112Z"},
**self.auth_headers)
assert response.status_code == 400
assert response.json() == {
'@event': 'QUERY_DID_NOT_VALIDATE',
'@type': 'error',
'errors': {
'to_sign': ['This field is required.']
},
'@authorizer': {
'account_type': 'PREMIUM',
'user_id': self.user_id,
}
}
# -- missing datetime
response = self.app.get(self.uri,
data={"to_sign": "hi there"},
**self.auth_headers)
assert response.status_code == 400
assert response.json() == {
'@event': 'QUERY_DID_NOT_VALIDATE',
'@type': 'error',
'errors': {
'datetime': ['This field is required.']
},
'@authorizer': {
'account_type': 'PREMIUM',
'user_id': self.user_id,
},
}
def test_get_400__broken_datetime(self):
response = self.app.get(self.uri,
data={
"to_sign": "hi there",
"datetime": "2017000001201T123112Z",
},
**self.auth_headers)
assert response.status_code == 400
assert response.json() == {
'@event': 'QUERY_DID_NOT_VALIDATE',
'@type': 'error',
'errors': {
'datetime':
[('invalid datetime format accepted is YYYYMMDDThhmmssZ')]
},
'@authorizer': {
'account_type': 'PREMIUM',
'user_id': self.user_id,
},
}
class CatalogueItemElementCommandsTestCase(TestCase):
def get_uri(self, item_id):
return reverse('catalogue:items.element', kwargs={'item_id': item_id})
def setUp(self):
ef.clear()
self.app = Client()
self.account = ef.account(type=AccountType.ADMIN.value)
token = AuthToken.encode(self.account)
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
#
# READ
#
def test_get_200(self):
ci_0 = ef.catalogue_item(name='temperatures')
# -- noise
ci_1 = ef.catalogue_item(name='iot_features') # noqa
response = self.app.get(self.get_uri(ci_0.id), **self.headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'CATALOGUEITEM_READ',
**CatalogueItemSerializer(ci_0).data,
}
def test_get_404(self):
response = self.app.get(self.get_uri(69506), **self.headers)
assert response.status_code == 404
assert response.json() == {
'@event': 'COULD_NOT_FIND_CATALOGUEITEM',
'@type': 'error',
'@authorizer': {
'account_id': self.account.id,
},
}
#
# UPDATE
#
def test_put_200(self):
a_0 = ef.account()
a_1 = ef.account()
ci = ef.catalogue_item(name='temperatures',
created_by=a_1,
updated_by=a_1)
response = self.app.put(self.get_uri(ci.id),
data=json.dumps({
'sample': [],
'spec': [
{
'name': 'value',
'type': 'FLOAT',
'size': 19203,
'is_nullable': False,
'is_enum': False,
'distribution': None,
},
],
'maintained_by_id':
a_0.id,
'executor_type':
'DATABRICKS',
}),
content_type='application/json',
**self.headers)
assert response.status_code == 200
ci.refresh_from_db()
assert response.json() == {
'@event': 'CATALOGUEITEM_UPDATED',
**CatalogueItemSerializer(ci).data
}
assert ci.name == 'temperatures'
assert ci.spec == [
{
'name': 'value',
'type': 'FLOAT',
'size': 19203,
'is_nullable': False,
'is_enum': False,
'distribution': None,
},
]
assert ci.maintained_by == a_0
assert ci.created_by == a_1
assert ci.updated_by == self.account
assert ci.executor_type == 'DATABRICKS'
def test_put_400(self):
a = ef.account()
ci = ef.catalogue_item(name='temperatures')
response = self.app.put(self.get_uri(ci.id),
data=json.dumps({
'name':
'iot_events',
'sample': [],
'spec': [
{
'name': 'value',
'size': 19203,
'is_nullable': False,
'is_enum': False,
'distribution': None,
},
],
'maintained_by_id':
a.id,
'executor_type':
'DATABRICKS',
}),
content_type='application/json',
**self.headers)
assert response.status_code == 400
assert response.json() == {
'@event': 'BODY_DID_NOT_VALIDATE',
'@type': 'error',
'errors': {
'spec': [
"JSON did not validate. PATH: '0' REASON: 'type' is a "
"required property",
],
},
'@authorizer': {
'account_id': self.account.id,
},
}
def test_put_404(self):
response = self.app.put(self.get_uri(9022),
data=json.dumps({
'name':
'iot_events',
'sample': [],
'spec': [
{
'name': 'value',
'type': 'FLOAT',
'size': 19203,
'is_nullable': False,
'is_enum': False,
'distribution': None,
},
],
'maintained_by_id':
ef.account().id,
'executor_type':
'DATABRICKS',
}),
content_type='application/json',
**self.headers)
assert response.status_code == 404
assert response.json() == {
'@event': 'COULD_NOT_FIND_CATALOGUEITEM',
'@type': 'error',
'@authorizer': {
'account_id': self.account.id,
},
}
#
# DELETE
#
def test_delete_200(self):
ci_0 = ef.catalogue_item(name='temperatures')
# -- noise
ci_1 = ef.catalogue_item(name='iot_features') # noqa
assert CatalogueItem.objects.all().count() == 2
response = self.app.delete(self.get_uri(ci_0.id), **self.headers)
assert CatalogueItem.objects.all().count() == 1
assert response.status_code == 200
assert response.json() == {
'@event': 'CATALOGUEITEM_DELETED',
'@type': 'empty',
}
def test_delete_400__not_cancelled_download_requests(self):
ci_0 = ef.catalogue_item(name='temperatures',
spec=[
{
'name': 'price',
'type': 'INTEGER',
'is_nullable': True,
'is_enum': True,
'distribution': None,
'size': 1920,
},
])
ef.download_request(spec={
'columns': ['price'],
'filters': [],
'randomize_ratio': 0.1,
},
catalogue_item=ci_0)
ef.download_request(spec={
'columns': ['price'],
'filters': [],
'randomize_ratio': 0.2,
},
catalogue_item=ci_0,
is_cancelled=True)
# -- noise
ci_1 = ef.catalogue_item(name='iot_features') # noqa
assert CatalogueItem.objects.all().count() == 2
response = self.app.delete(self.get_uri(ci_0.id), **self.headers)
assert CatalogueItem.objects.all().count() == 2
assert response.status_code == 400
assert response.json() == {
'@authorizer': {
'account_id': self.account.id
},
'@event': 'NOT_CANCELLED_DOWNLOAD_REQEUSTS_DETECTED',
'@type': 'error',
'item_id': ci_0.id,
'not_cancelled_count': 1,
}
def test_delete_404(self):
response = self.app.delete(self.get_uri(69506), **self.headers)
assert response.status_code == 404
assert response.json() == {
'@event': 'COULD_NOT_FIND_CATALOGUEITEM',
'@type': 'error',
'@authorizer': {
'account_id': self.account.id,
},
}
class CatalogueItemCollectionCommandsTestCase(TestCase):
uri = reverse('catalogue:items.collection')
def setUp(self):
ef.clear()
self.app = Client()
self.account = ef.account(type=AccountType.ADMIN.value)
token = AuthToken.encode(self.account)
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
#
# CREATE CATALOGUE ITEM
#
def test_post_201(self):
a = ef.account()
assert CatalogueItem.objects.all().count() == 0
response = self.app.post(self.uri,
data=json.dumps({
'name':
'iot_events',
'sample': [],
'spec': [
{
'name': 'value',
'type': 'FLOAT',
'size': 19203,
'is_nullable': False,
'is_enum': False,
'distribution': None,
},
],
'maintained_by_id':
a.id,
'executor_type':
'DATABRICKS',
}),
content_type='application/json',
**self.headers)
assert response.status_code == 201
assert CatalogueItem.objects.all().count() == 1
ci = CatalogueItem.objects.all().first()
assert response.json() == {
'@event': 'CATALOGUEITEM_CREATED',
**CatalogueItemSerializer(ci).data,
}
assert ci.created_by == self.account
assert ci.updated_by == self.account
def test_post_400__broken_request(self):
a = ef.account()
assert CatalogueItem.objects.all().count() == 0
response = self.app.post(self.uri,
data=json.dumps({
'name':
'iot_events',
'sample': [],
'spec': [
{
'name': 'value',
'size': 19203,
'is_nullable': False,
'distribution': None,
},
],
'maintained_by_id':
a.id,
'executor_type':
'DATABRICKS',
}),
content_type='application/json',
**self.headers)
assert response.status_code == 400
assert CatalogueItem.objects.all().count() == 0
assert response.json() == {
'@event': 'BODY_DID_NOT_VALIDATE',
'@type': 'error',
'errors': {
'spec': [
"JSON did not validate. PATH: '0' REASON: 'type' is a "
"required property",
],
},
'@authorizer': {
'account_id': self.account.id,
},
}
def test_post_400__maintainer_does_not_exist(self):
assert CatalogueItem.objects.all().count() == 0
response = self.app.post(self.uri,
data=json.dumps({
'name':
'iot_events',
'sample': [],
'spec': [
{
'name': 'value',
'type': 'FLOAT',
'size': 19203,
'is_nullable': False,
'is_enum': False,
'distribution': None,
},
],
'maintained_by_id':
932039,
'executor_type':
'DATABRICKS',
}),
content_type='application/json',
**self.headers)
assert response.status_code == 400
assert CatalogueItem.objects.all().count() == 0
assert response.json() == {
'errors': {
'maintained_by':
['account instance with id 932039 does not exist.'],
},
'@authorizer': {
'account_id': self.account.id,
},
'@type': 'error',
'@event': 'BODY_JSON_DID_NOT_PARSE',
}
#
# BULK READ CATALOGUE ITEMS
#
def test_get_200(self):
ci_0 = ef.catalogue_item(name='iot_features')
ci_1 = ef.catalogue_item(name='iot_events')
ci_2 = ef.catalogue_item(name='temperatures')
response = self.app.get(self.uri, **self.headers)
assert response.status_code == 200
assert response.json() == {
'@event':
'CATALOGUEITEMS_BULK_READ',
'@type':
'catalogue_items_list',
'items': [
CatalogueItemSerializer(ci_0).data,
CatalogueItemSerializer(ci_1).data,
CatalogueItemSerializer(ci_2).data,
],
}
def test_get_200__with_query(self):
ci_0 = ef.catalogue_item(name='iot_features')
ci_1 = ef.catalogue_item(name='temperatures') # noqa
ci_2 = ef.catalogue_item(name='iot_events')
response = self.app.get(self.uri,
data={'query': 'IoT'},
**self.headers)
assert response.status_code == 200
assert response.json() == {
'@event':
'CATALOGUEITEMS_BULK_READ',
'@type':
'catalogue_items_list',
'items': [
CatalogueItemSerializer(ci_0).data,
CatalogueItemSerializer(ci_2).data,
],
}
def test_get_200__query_many_words_default(self):
ci_0 = ef.catalogue_item(name='iot_features')
ci_1 = ef.catalogue_item(name='temperatures') # noqa
ci_2 = ef.catalogue_item(name='iot_events') # noqa
response = self.app.get(self.uri,
data={'query': 'feature tempera'},
**self.headers)
assert response.status_code == 200
assert response.json() == {
'@event':
'CATALOGUEITEMS_BULK_READ',
'@type':
'catalogue_items_list',
'items': [
CatalogueItemSerializer(ci_0).data,
CatalogueItemSerializer(ci_1).data,
],
}
def test_get_200__query_many_words_and_or(self):
ci_0 = ef.catalogue_item(name='iot_features')
ci_1 = ef.catalogue_item(name='temperatures') # noqa
ci_2 = ef.catalogue_item(name='iot_events') # noqa
response = self.app.get(self.uri,
data={'query': 'feature & IOT | temperature'},
**self.headers)
assert response.status_code == 200
assert response.json() == {
'@event':
'CATALOGUEITEMS_BULK_READ',
'@type':
'catalogue_items_list',
'items': [
CatalogueItemSerializer(ci_0).data,
CatalogueItemSerializer(ci_1).data,
],
}
def test_get_200__query_many_words_or_not(self):
ci_0 = ef.catalogue_item(name='iot_features') # noqa
ci_1 = ef.catalogue_item(name='temperatures') # noqa
ci_2 = ef.catalogue_item(name='iot_events')
response = self.app.get(self.uri,
data={'query': 'IOT ~features | temp'},
**self.headers)
assert response.status_code == 200
assert response.json() == {
'@event':
'CATALOGUEITEMS_BULK_READ',
'@type':
'catalogue_items_list',
'items': [
CatalogueItemSerializer(ci_1).data,
CatalogueItemSerializer(ci_2).data,
],
}
def test_get_200__with_has_samples(self):
ci_0 = ef.catalogue_item(name='iot_features', sample=[])
ci_1 = ef.catalogue_item(name='temperatures',
sample=[
{
'location': 'Wroclaw',
'value': 12.1
},
{
'location': 'Olawa',
'value': 34.4
},
])
ci_2 = ef.catalogue_item(name='iot_events', sample=[])
# -- with samples
response = self.app.get(self.uri,
data={'has_samples': True},
**self.headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'CATALOGUEITEMS_BULK_READ',
'@type': 'catalogue_items_list',
'items': [
CatalogueItemSerializer(ci_1).data,
],
}
# -- without samples
response = self.app.get(self.uri,
data={'has_samples': False},
**self.headers)
assert response.status_code == 200
assert response.json() == {
'@event':
'CATALOGUEITEMS_BULK_READ',
'@type':
'catalogue_items_list',
'items': [
CatalogueItemSerializer(ci_0).data,
CatalogueItemSerializer(ci_2).data,
],
}
class EntryPointCommandsTestCase(TestCase):
uri = reverse('entrypoint:entrypoint')
@pytest.fixture(autouse=True)
def initfixtures(self, mocker, tmpdir):
self.mocker = mocker
self.tmpdir = tmpdir
def setUp(self):
self.app = Client()
self.auth_headers = {
'HTTP_X_CS_ACCOUNT_TYPE': 'ADMIN',
'HTTP_X_CS_USER_ID': 190,
}
self.lily_dir = self.tmpdir.mkdir('.lily')
self.commands_dir = self.lily_dir.mkdir('commands')
MockConfig._lily_path = str(self.lily_dir)
self.mocker.patch(
'entrypoint.commands.Config', MockConfig)
def test_get(self):
c = ef.command()
self.mocker.patch.object(
EntryPointCommands,
'get_commands'
).return_value = {
'enums': [],
'UPDATE_HELLO': CommandSerializer(deepcopy(c)).data,
}
response = self.app.get(self.uri, **self.auth_headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'ENTRY_POINT_READ',
'@type': 'entrypoint',
'enums': [],
'name': 'test',
'version_info': {
'@type': 'version_info',
'available': [],
'deployed': '2.5.6',
'displayed': '2.5.6',
},
'commands': {
'UPDATE_HELLO': CommandSerializer(c).data,
},
}
def test_get__with_versions(self):
c = ef.command()
self.mocker.patch.object(
EntryPointCommands,
'get_commands'
).return_value = {
'enums': [],
'UPDATE_HELLO': CommandSerializer(deepcopy(c)).data,
}
self.commands_dir.join('2.5.6.json').write('..')
self.commands_dir.join('2.14.5.json').write('..')
self.commands_dir.join('2.120.0.json').write('..')
self.commands_dir.join('1.0.0.json').write('..')
response = self.app.get(self.uri, **self.auth_headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'ENTRY_POINT_READ',
'@type': 'entrypoint',
'enums': [],
'name': 'test',
'version_info': {
'@type': 'version_info',
'available': ['2.120.0', '2.14.5', '2.5.6', '1.0.0'],
'deployed': '2.5.6',
'displayed': '2.5.6',
},
'commands': {
'UPDATE_HELLO': CommandSerializer(c).data,
},
}
def test_get__filter_by_commands_query(self):
c0, c1, c2 = ef.command(), ef.command(), ef.command()
self.mocker.patch.object(
EntryPointCommands,
'get_commands'
).side_effect = [
{
'enums': [],
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
'READ_PAYMENTS': CommandSerializer(deepcopy(c1)).data,
'FIND_TOOL': CommandSerializer(deepcopy(c2)).data,
},
{
'enums': [],
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
'READ_PAYMENTS': CommandSerializer(deepcopy(c1)).data,
'FIND_TOOL': CommandSerializer(deepcopy(c2)).data,
},
]
# -- filter two commands
response = self.app.get(
self.uri,
data={
'commands': ['UPDATE_HELLO', 'FIND_TOOL']
},
**self.auth_headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'ENTRY_POINT_READ',
'@type': 'entrypoint',
'enums': [],
'name': 'test',
'version_info': {
'@type': 'version_info',
'available': [],
'deployed': '2.5.6',
'displayed': '2.5.6',
},
'commands': {
'UPDATE_HELLO': CommandSerializer(c0).data,
'FIND_TOOL': CommandSerializer(c2).data,
},
}
# -- filter single command - triangulation
response = self.app.get(
self.uri,
data={
'commands': ['READ_PAYMENTS']
},
**self.auth_headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'ENTRY_POINT_READ',
'@type': 'entrypoint',
'enums': [],
'name': 'test',
'version_info': {
'@type': 'version_info',
'available': [],
'deployed': '2.5.6',
'displayed': '2.5.6',
},
'commands': {
'READ_PAYMENTS': CommandSerializer(c1).data,
},
}
def test_get__filter_by_is_private(self):
c0 = ef.command(is_private=True)
c1 = ef.command(is_private=False)
c2 = ef.command(is_private=True)
self.mocker.patch.object(
EntryPointCommands,
'get_commands'
).side_effect = [
{
'enums': [],
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
'CREATE_HELLO': CommandSerializer(deepcopy(c1)).data,
'DELETE_HELLO': CommandSerializer(deepcopy(c2)).data,
},
{
'enums': [],
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
'CREATE_HELLO': CommandSerializer(deepcopy(c1)).data,
'DELETE_HELLO': CommandSerializer(deepcopy(c2)).data,
},
]
# -- show only private commands
response = self.app.get(
self.uri,
data={'is_private': True},
**self.auth_headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'ENTRY_POINT_READ',
'@type': 'entrypoint',
'enums': [],
'name': 'test',
'version_info': {
'@type': 'version_info',
'available': [],
'deployed': '2.5.6',
'displayed': '2.5.6',
},
'commands': {
'UPDATE_HELLO': CommandSerializer(c0).data,
'DELETE_HELLO': CommandSerializer(c2).data,
},
}
# -- show only public commands
response = self.app.get(
self.uri,
data={'is_private': False},
**self.auth_headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'ENTRY_POINT_READ',
'@type': 'entrypoint',
'enums': [],
'name': 'test',
'version_info': {
'@type': 'version_info',
'available': [],
'deployed': '2.5.6',
'displayed': '2.5.6',
},
'commands': {
'CREATE_HELLO': CommandSerializer(c1).data,
},
}
def test_get__filter_by_domain_id(self):
c0 = ef.command(domain_id='cards')
c1 = ef.command(domain_id='paths')
c2 = ef.command(domain_id='paths')
self.mocker.patch.object(
EntryPointCommands,
'get_commands'
).side_effect = [
{
'enums': [{'A': 'C'}],
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
'CREATE_HELLO': CommandSerializer(deepcopy(c1)).data,
'DELETE_HELLO': CommandSerializer(deepcopy(c2)).data,
},
{
'enums': [{'A': 'X'}],
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
'CREATE_HELLO': CommandSerializer(deepcopy(c1)).data,
'DELETE_HELLO': CommandSerializer(deepcopy(c2)).data,
},
]
# -- show CARDS domain commands
response = self.app.get(
self.uri,
data={'domain_id': 'cards'},
**self.auth_headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'ENTRY_POINT_READ',
'@type': 'entrypoint',
'enums': [{'A': 'C'}],
'name': 'test',
'version_info': {
'@type': 'version_info',
'available': [],
'deployed': '2.5.6',
'displayed': '2.5.6',
},
'commands': {
'UPDATE_HELLO': CommandSerializer(c0).data,
},
}
# -- show PATHS domain commands
response = self.app.get(
self.uri,
data={'domain_id': 'PATHS'},
**self.auth_headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'ENTRY_POINT_READ',
'@type': 'entrypoint',
'enums': [{'A': 'X'}],
'name': 'test',
'version_info': {
'@type': 'version_info',
'available': [],
'deployed': '2.5.6',
'displayed': '2.5.6',
},
'commands': {
'CREATE_HELLO': CommandSerializer(c1).data,
'DELETE_HELLO': CommandSerializer(c2).data,
},
}
def test_get__version(self):
c = ef.command()
get_commands = self.mocker.patch.object(
EntryPointCommands, 'get_commands')
get_commands.return_value = {
'enums': [],
'UPDATE_HELLO': CommandSerializer(deepcopy(c)).data,
}
response = self.app.get(
self.uri,
data={'version': '2.1.3'},
**self.auth_headers)
assert response.status_code == 200
version_info = response.json()['version_info']
assert version_info == {
'@type': 'version_info',
'available': [],
'deployed': '2.5.6',
'displayed': '2.1.3',
}
assert get_commands.call_args_list == [call('2.1.3')]
#
# GET_COMMANDS
#
def test_get_commands(self):
c0 = ef.command(domain_id='cards')
c1 = ef.command(domain_id='paths')
commands = {
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
'CREATE_HELLO': CommandSerializer(deepcopy(c1)).data,
}
self.commands_dir.join('2.5.6.json').write(json.dumps(commands))
assert EntryPointCommands().get_commands() == commands
def test_get_commands__version(self):
c0 = ef.command(domain_id='cards')
c1 = ef.command(domain_id='paths')
commands0 = { # noqa
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
'CREATE_HELLO': CommandSerializer(deepcopy(c1)).data,
}
self.commands_dir.join('2.5.6.json').write(json.dumps(commands0))
commands1 = {
'UPDATE_HELLO': CommandSerializer(deepcopy(c0)).data,
}
self.commands_dir.join('2.0.0.json').write(json.dumps(commands1))
assert EntryPointCommands().get_commands('2.0.0') == commands1
def test_multiple_responses_for_multiple_endpoints(self):
examples_file = self.prepare_example_file()
Client().get('/test/it/?type=a')
Client().get('/test/it/?type=b')
Client().get('/test/it/?type=a')
Client().delete('/test/it/')
Client().get('/test/it/?type=d')
Client().put('/test/it/')
assert json.loads(examples_file.read()) == {
'GET_IT': {
'200 (LISTED)': {
'method': 'get',
'description': 'LISTED',
'request': {
'path': '/test/it/?type=a'
},
'response': {
'status': 200,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'LISTED',
'hello': 'get.a',
},
}
},
'404 (ERROR_LISTED)': {
'method': 'get',
'description': 'ERROR_LISTED',
'request': {
'path': '/test/it/?type=d'
},
'response': {
'status': 404,
'content_type': 'application/json',
'content': {
'@type': 'error',
'@event': 'ERROR_LISTED',
'hello': 'get.d',
},
}
},
},
'DELETE_IT': {
'200 (DELETED)': {
'method': 'delete',
'description': 'DELETED',
'request': {
'path': '/test/it/'
},
'response': {
'status': 200,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'DELETED',
'hello': 'delete',
},
},
}
},
'PUT_IT': {
'200 (UPDATED)': {
'method': 'put',
'description': 'UPDATED',
'request': {
'path': '/test/it/'
},
'response': {
'status': 200,
'content_type': 'application/json',
'content': {
'@type': 'test',
'@event': 'UPDATED',
'hello': 'put',
},
},
}
},
}
def setUp(self):
ef.clear()
self.app = Client()
class DownloadRequestCollectionCommandsTestCase(TestCase):
uri = reverse('downloader:requests.collection')
@pytest.fixture(autouse=True)
def initfixtures(self, mocker):
self.mocker = mocker
def setUp(self):
ef.clear()
self.app = Client()
self.account = ef.account(type=AccountType.ADMIN.value)
self.ci = ef.catalogue_item(spec=[
{
'name': 'product',
'type': 'STRING',
'is_nullable': True,
'is_enum': True,
'size': None,
'distribution': None,
},
{
'name': 'price',
'type': 'INTEGER',
'is_nullable': False,
'is_enum': False,
'size': None,
'distribution': None,
},
],
executor_type='ATHENA')
token = AuthToken.encode(self.account)
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
#
# CREATE DOWNLOAD REQUEST
#
def test_post_201__created(self):
execute = self.mocker.patch.object(AthenaExecutor, 'execute')
execute.return_value = (
'https://s3.this.region.amazonaws.com/buk.et/results/567.csv')
assert DownloadRequest.objects.all().count() == 0
response = self.app.post(self.uri,
data=json.dumps({
'spec': {
'columns': ['product', 'price'],
'filters': [],
'randomize_ratio': 0.9,
},
'catalogue_item_id':
self.ci.id,
}),
content_type='application/json',
**self.headers)
assert response.status_code == 201
assert DownloadRequest.objects.all().count() == 1
r = DownloadRequest.objects.all().first()
assert response.json() == {
'@event': 'DOWNLOADREQUEST_CREATED',
**DownloadRequestSerializer(r).data,
}
assert r.created_by == self.account
assert r.blob_name == (
'https://s3.this.region.amazonaws.com/buk.et/results/567.csv')
assert execute.call_args_list == [call(r)]
def test_post_200__read(self):
execute = self.mocker.patch.object(AthenaExecutor, 'execute')
r = ef.download_request(
spec={
'columns': ['price', 'product'],
'filters': [
{
'name': 'price',
'operator': '>=',
'value': 78
},
{
'name': 'price',
'operator': '=',
'value': 23
},
{
'name': 'product',
'operator': '=',
'value': 'jack'
},
],
'randomize_ratio':
0.9,
},
blob_name=(
'https://s3.this.region.amazonaws.com/buk.et/results/567.csv'),
catalogue_item=self.ci)
assert DownloadRequest.objects.all().count() == 1
response = self.app.post(self.uri,
data=json.dumps({
'spec': {
'columns': ['product', 'price'],
'filters': [
{
'name': 'product',
'operator': '=',
'value': 'jack'
},
{
'name': 'price',
'operator': '=',
'value': 23
},
{
'name': 'price',
'operator': '>=',
'value': 78
},
],
'randomize_ratio':
0.9,
},
'catalogue_item_id':
self.ci.id,
}),
content_type='application/json',
**self.headers)
assert response.status_code == 200
assert DownloadRequest.objects.all().count() == 1
assert DownloadRequest.objects.all().first() == r
assert response.json() == {
'@event': 'DOWNLOADREQUEST_READ',
**DownloadRequestSerializer(r).data,
}
assert execute.call_count == 0
def test_post_400__broken_request(self):
assert DownloadRequest.objects.all().count() == 0
response = self.app.post(self.uri,
data=json.dumps({
'spec': {
'columns': ['product', 'price'],
'filters': [],
'randomize_ratio': 0.9,
},
'catalogue_item_id': 'TEXT',
}),
content_type='application/json',
**self.headers)
assert response.status_code == 400
assert DownloadRequest.objects.all().count() == 0
assert response.json() == {
'@event': 'BODY_DID_NOT_VALIDATE',
'@type': 'error',
'errors': {
'catalogue_item_id': ['A valid integer is required.'],
},
'@authorizer': {
'account_id': self.account.id,
},
}
def test_post_400__catalogue_item_does_not_exist(self):
assert DownloadRequest.objects.all().count() == 0
response = self.app.post(self.uri,
data=json.dumps({
'spec': {
'columns': ['product', 'price'],
'filters': [],
'randomize_ratio': 0.9,
},
'catalogue_item_id': 58495,
}),
content_type='application/json',
**self.headers)
assert response.status_code == 400
assert DownloadRequest.objects.all().count() == 0
assert response.json() == {
'errors': {
'catalogue_item':
['catalogue item instance with id 58495 does not exist.']
},
'@event': 'BODY_JSON_DID_NOT_PARSE',
'@type': 'error',
'@authorizer': {
'account_id': self.account.id,
},
}
#
# BULK READ DOWNLOAD REQUESTS
#
def test_get_200(self):
a = ef.account()
d_0 = DownloadRequest.objects.create(created_by=a,
spec={
'columns': ['product'],
'filters': [{
'name': 'price',
'operator': '>=',
'value': 78
}],
'randomize_ratio':
1,
},
catalogue_item=self.ci)
d_0.waiters.add(self.account)
d_1 = DownloadRequest.objects.create(created_by=a,
spec={
'columns': ['product'],
'filters': [{
'name': 'price',
'operator': '=',
'value': 18
}],
'randomize_ratio':
0.8,
},
catalogue_item=self.ci)
d_1.waiters.add(self.account)
# -- noise
d_2 = DownloadRequest.objects.create( # noqa
created_by=a,
spec={
'columns': ['price'],
'filters': [],
'randomize_ratio': 0.1,
},
catalogue_item=self.ci)
response = self.app.get(self.uri, **self.headers)
assert response.status_code == 200
assert response.json() == {
'@event':
'DOWNLOADREQUESTS_BULK_READ',
'@type':
'download_requests_list',
'requests': [
DownloadRequestSerializer(d_0).data,
DownloadRequestSerializer(d_1).data,
],
}
class DownloadRequestElementCommandsTestCase(TestCase):
def get_uri(self, request_id):
return reverse('downloader:requests.element',
kwargs={'request_id': request_id})
def setUp(self):
ef.clear()
self.app = Client()
self.account = ef.account(type=AccountType.ADMIN.value)
token = AuthToken.encode(self.account)
self.headers = {'HTTP_AUTHORIZATION': f'Bearer {token}'}
self.ci = ef.catalogue_item(spec=[
{
'name': 'product',
'type': 'STRING',
'is_nullable': True,
'is_enum': True,
'size': None,
'distribution': None,
},
{
'name': 'price',
'type': 'INTEGER',
'is_nullable': False,
'is_enum': False,
'size': None,
'distribution': None,
},
])
#
# READ
#
def test_get_200(self):
d = DownloadRequest.objects.create(created_by=ef.account(),
spec={
'columns': ['product'],
'filters': [{
'name': 'price',
'operator': '>=',
'value': 78
}],
'randomize_ratio':
1,
},
catalogue_item=self.ci)
d.waiters.add(self.account)
response = self.app.get(self.get_uri(d.id), **self.headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'DOWNLOADREQUEST_READ',
**DownloadRequestSerializer(d).data,
}
def test_get_404__wrong_user(self):
d = DownloadRequest.objects.create(created_by=ef.account(),
spec={
'columns': ['product'],
'filters': [{
'name': 'price',
'operator': '>=',
'value': 78
}],
'randomize_ratio':
1,
},
catalogue_item=self.ci)
response = self.app.get(self.get_uri(d.id), **self.headers)
assert response.status_code == 404
assert response.json() == {
'@event': 'COULD_NOT_FIND_DOWNLOADREQUEST',
'@type': 'error',
'@authorizer': {
'account_id': self.account.id,
},
}
def test_get_404__wrong_id(self):
response = self.app.get(self.get_uri(69506), **self.headers)
assert response.status_code == 404
assert response.json() == {
'@event': 'COULD_NOT_FIND_DOWNLOADREQUEST',
'@type': 'error',
'@authorizer': {
'account_id': self.account.id,
},
}
#
# DELETE
#
def test_delete_200(self):
d = DownloadRequest.objects.create(created_by=ef.account(),
spec={
'columns': ['product'],
'filters': [{
'name': 'price',
'operator': '>=',
'value': 78
}],
'randomize_ratio':
1,
},
is_cancelled=False,
catalogue_item=self.ci)
d.waiters.add(self.account)
assert DownloadRequest.objects.all().count() == 1
response = self.app.delete(self.get_uri(d.id), **self.headers)
assert response.status_code == 200
assert response.json() == {
'@event': 'DOWNLOADREQUEST_DELETED',
'@type': 'empty',
}
assert DownloadRequest.objects.all().count() == 1
d.refresh_from_db()
assert d.waiters.count() == 0
assert d.is_cancelled is True
def test_delete_404(self):
response = self.app.delete(self.get_uri(69506), **self.headers)
assert response.status_code == 404
assert response.json() == {
'@event': 'COULD_NOT_FIND_DOWNLOADREQUEST',
'@type': 'error',
'@authorizer': {
'account_id': self.account.id,
},
}
