def test_get_non_existent_user_id(self):
"""
REST API: http://<gateway>/api/v2/users
METHOD: GET
tests getting non-existent user returns an error.
"""
test_name = 'test_get_non_existent_user_id '
user_id = 'doesnotexist'
expected_status = 404
expected_error_message = 'user not found'
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Get non-existent user id')
status, actual_user_id, actual_user_name, error_message = \
gateway_util.get_user_by_id(self, self.gateway, user_id)
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status \'%s\'' %
(status, expected_status))
_assert(self,
status,
expected_status,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
self.mylog.info(
test_name +
'Assert actual error message \'%s\' equals to expected error message \'%s\''
% (error_message, expected_error_message))
_assert(self, error_message, expected_error_message, 'error message')
def test_get_non_existent_bucket_id_16_bytes(self):
"""
REST API: http://<gateway>/api/v2/buckets
METHOD: GET
tests getting non-existent bucket returns an error.
"""
test_name = 'test_get_non_existent_bucket_id '
bucket_id = '9999999999999999'
expected_status = 500
expected_error_message = 'bucket not found'
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Get non-existent bucket id')
get_bucket_result = buckets_util.get_bucket_by_id(
self, self.gateway, bucket_id)
status = get_bucket_result['status']
error_message = get_bucket_result['error_message']
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status \'%s\'' %
(status, expected_status))
_assert(self,
status,
expected_status,
'status code',
xfail=True,
reason='status code is \'%s\'' % status)
self.mylog.info(
test_name +
'Assert actual error message \'%s\' equals to expected error message \'%s\''
% (error_message, expected_error_message))
_assert(self, error_message, expected_error_message, '')
def test_get_non_existent_org_id_16_bytes(self):
"""
REST API: http://<gateway>/api/v2/orgs
METHOD: GET
tests getting non-existent org returns an error.
"""
test_name = 'test_get_non_existent_org_id '
org_id = '9999999999999999'
expected_status = 500
expected_error_message = 'organization not found'
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Get non-existent org id')
result = org_util.get_organization_by_id(self, self.gateway, org_id)
status = result['status']
error_message = result['error_message']
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status \'%s\'' %
(status, expected_status))
_assert(self,
status,
expected_status,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
self.mylog.info(
test_name +
'Assert actual error message \'%s\' equals to expected error message \'%s\''
% (error_message, expected_error_message))
_assert(self, error_message, expected_error_message, 'error message')
def test_create_bucket_no_org_id(self):
"""
REST API: http://<gateway>/api/v2/buckets
METHOD: POST
tests cannot create bucket if organization id is not provided
"""
test_name = 'test_create_bucket_no_org_id'
bucket_name = 'bucket_no_org_id'
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Create Bucket \'%s\' without ORG ID' % bucket_name)
create_bucket_result = buckets_util.create_bucket(self, self.gateway, bucket_name, 3600)
status = create_bucket_result['status']
_assert(self, status, 404, 'status code', xfail=True, reason='status=%s' % status)
def test_verify_created_users(self):
"""
REST API: http://<gateway>/api/v2/users
METHOD: GET
tests that created user can be found in the list of all users returned by the 'get all users' endpoint
"""
test_name = 'test_verify_created_users '
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: find users by name')
for name in user_names:
success = gateway_util.find_user_by_name(self, name,
self.get_all_setup_users)
_assert(self, success, True, 'find user by name')
self.footer(test_name)
def test_verify_created_orgs(self):
"""
REST API: http://<gateway>/api/v2/orgs
METHOD: GET
tests that created org can be found in the list of all orgs returned by the 'get all orgs' endpoint
"""
test_name = 'test_verify_created_orgs '
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: find organization by name')
for name in org_names:
org_name, org_id = org_util.find_org_by_name(
self, name, self.get_all_setup_orgs)
self.mylog.info(test_name + 'Assert expected name ' + str(name) +
' equals to actual name ' + str(org_name))
_assert(self, org_name, name, 'org name')
self.footer(test_name)
def test_update_orgs_empty_name(self):
"""
REST API: http://<gateway>/api/v2/orgs
METHOD: PATCH
tests org name can be updated and persisted in the etcd store if name is empty
"""
test_name = 'test_update_orgs_empty_name'
org_name = 'org_to_be_updated'
org_name_to_update = ''
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create Organization')
create_result = org_util.create_organization(self, self.gateway,
org_name)
status = create_result['status']
created_org_id = create_result['org_id']
created_org_name = create_result['org_name']
self.mylog.info(
test_name +
'Assert actual status code \'%s\' equals to expected status code 201'
% status)
_assert(self, status, 201, 'status_code')
self.mylog.info(
test_name +
'STEP 2: Verify created organization was persisted in the etcd store'
)
verify_org_etcd_entries(self,
test_name,
created_org_id,
created_org_name,
error='')
self.mylog.info(test_name +
' STEP 3: Update created org with the \'%s\' name' %
org_name_to_update)
update_result = org_util.update_organization(self, self.gateway,
created_org_id,
org_name_to_update)
status = update_result['status']
updated_org_id = update_result['updated_org_id']
updated_org_name = update_result['updated_org_name']
self.mylog.info(
test_name +
'Assert actual status code \'%s\' equals to expected status code 200'
% status)
_assert(self, status, 200, 'status_code')
self.mylog.info(
test_name +
'Assert updated org_id \'%s\' equals to expected org_id \'%s\'' %
(updated_org_id, created_org_id))
_assert(self, updated_org_id, created_org_id, 'org_id')
self.mylog.info(
test_name +
'Assert updated org_name \'%s\' equals to expected org_name \'%s\''
% (updated_org_name, ''))
_assert(self, updated_org_name, '', 'org_name')
self.footer(test_name)
def test_update_users_empty_name(self):
"""
REST API: http://<gateway>/api/v2/users
METHOD: PATCH
tests user name can be updated and persisted in the etcd store if name is empty
"""
test_name = 'test_update_users_empy_name'
user_name = 'user_to_be_updated'
user_name_to_update = ''
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create User \'%s\'' % user_name)
status, created_user_id, created_user_name, error_message = \
gateway_util.create_user(self, self.gateway, user_name)
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status \'%s\'' %
(status, 201))
_assert(self, status, 201, 'status code')
self.mylog.info(test_name +
'STEP 2: Verify data was persisted in the etcd store')
verify_user_etcd_entries(self,
test_name,
created_user_id,
created_user_name,
expected_error='')
self.mylog.info(test_name +
' STEP 3: Update created user with the \'%s\' name' %
user_name_to_update)
status, updated_user_id, updated_user_name, error_message = \
gateway_util.update_user(self, self.gateway, created_user_id, user_name_to_update)
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status \'%s\'' %
(status, 200))
_assert(self, status, 200, 'status code')
self.mylog.info(
test_name +
'Assert updated user_id \'%s\' equals to expected user_id \'%s\'' %
(updated_user_id, created_user_id))
_assert(self, updated_user_id, created_user_id, 'user id')
self.mylog.info(
test_name +
'Assert updated user_name \'%s\' equals to expected user_name \'%s\''
% (updated_user_name, user_name_to_update))
_assert(self, updated_user_name, user_name_to_update, 'user name')
self.mylog.info(test_name +
'STEP 4: Verify data was persisted in the etcd store')
verify_user_etcd_entries(self,
test_name,
updated_user_id,
updated_user_name,
expected_error='')
self.footer(test_name)
def test_get_all_buckets_count(self):
"""
REST API: http://<gateway>/api/v2/buckets
METHOD: GET
tests that the count of the created buckets equals to expected
"""
test_name = 'test_get_all_buckets_count '
expected_buckets_count = 130
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Get count of all buckets')
actual_count = buckets_util.get_count_of_buckets(
self, self.get_all_setup_buckets)
self.mylog.info(test_name + 'Actual count of created buckets is ' +
str(actual_count))
self.mylog.info(test_name + 'Assert expected_count ' +
str(expected_buckets_count) +
' equals to actual count ' + str(actual_count))
self.footer(test_name)
_assert(self, actual_count, expected_buckets_count, 'buckets count')
def test_verify_created_buckets(self):
"""
REST API: http://<gateway>/api/v2/buckets
METHOD: GET
tests that created bucket can be found in the list of all buckets returned by the 'get all buckets' endpoint
"""
test_name = 'test_verify_created_buckets '
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: find bucket per org')
for org_name in org_names:
for bucket_name in ascii_uppercase:
success = buckets_util.find_bucket_by_name(
self, self.get_all_setup_buckets, bucket_name, org_name)
self.mylog.info(
test_name +
'Assert actual bucket \'%s\' could be found in \'%s\'' %
(bucket_name, org_name))
_assert(self, success, True, 'find bucket per org')
self.footer(test_name)
def test_get_all_orgs_count(self):
"""
REST API: http://<gateway>/api/v2/orgs
METHOD: GET
tests that the count of the created orgs equals to expected
"""
test_name = 'test_get_all_orgs_count '
expected_orgs_count = 5
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Get count of all organizations')
actual_count = org_util.get_count_of_orgs(self,
self.get_all_setup_orgs)
self.mylog.info(test_name +
'Actual count of created organizations is ' +
str(actual_count))
self.mylog.info(test_name + 'Assert expected_count ' +
str(expected_orgs_count) + ' equals to actual count ' +
str(actual_count))
_assert(self, actual_count, expected_orgs_count, 'orgs count')
self.footer(test_name)
def test_get_all_users_count(self):
"""
REST API: http://<gateway>/api/v2/users
METHOD: GET
tests that the count of the created users equals to expected
"""
test_name = 'test_get_all_users_count '
expected_users_count = 10
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Get count of all users')
actual_count = gateway_util.get_count_of_users(
self, self.get_all_setup_users)
self.mylog.info(test_name + 'Actual count of created users is ' +
str(actual_count))
self.mylog.info(
test_name +
'Assert actual user_count \'%s\' equals to expected user_count \'%s\''
% (actual_count, expected_users_count))
_assert(self, actual_count, expected_users_count, 'user count')
self.footer(test_name)
def test_update_orgs_already_exist(self):
"""
REST API: http://<gateway>/api/v2/users
METHOD: PATCH
tests org name can be updated and persisted in the etcd store if name already exists.
"""
test_name = 'test_update_org`_already_exist'
org_name = 'existing_org'
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create Organization')
create_result = org_util.create_organization(self, self.gateway,
org_name)
status = create_result['status']
created_org_id = create_result['org_id']
self.mylog.info(
test_name +
'Assert actual status code \'%s\' equals to expected status code 201'
% status)
_assert(self, status, 201, 'status_code')
self.mylog.info(test_name +
' STEP 2: Update created org with the \'%s\' name' %
org_name)
update_result = org_util.update_organization(self, self.gateway,
created_org_id, org_name)
status = update_result['status']
updated_org_id = update_result['updated_org_id']
updated_org_name = update_result['updated_org_name']
self.mylog.info(
test_name +
'Assert actual status code \'%s\' equals to expected status code 201'
% status)
_assert(self, status, 200, 'status_code')
self.mylog.info(
test_name +
'Assert updated org_id \'%s\' equals to expected org_id \'%s\'' %
(updated_org_id, created_org_id))
_assert(self, updated_org_id, created_org_id, 'org_id')
self.mylog.info(
test_name +
'Assert updated org_name \'%s\' equals to expected org_name \'%s\''
% (updated_org_name, org_name))
_assert(self, updated_org_name, org_name, 'org_name')
self.mylog.info(
test_name +
'STEP 3: Verify created user was persisted in the etcd store')
verify_org_etcd_entries(self, test_name, updated_org_id,
updated_org_name, '')
self.footer(test_name)
def test_create_same_bucket_different_orgs(self):
"""
REST API: http://<gateway>/api/v2/buckets
METHOD: POST
tests bucket with the same name can be created for different organizations
"""
bucket_name = 'one_for_all'
test_name = 'test_create_same_bucket_different_orgs '
self.header(test_name)
for org_name in ascii_lowercase:
org_name = org_name + '_same_bucket_name'
self.mylog.info(test_name + 'STEP 1: Create Organization \'%s\'' % org_name)
create_org_result = org_util.create_organization(self, self.gateway, org_name)
status = create_org_result['status']
created_org_id = create_org_result['org_id']
created_org_name = create_org_result['org_name']
self.mylog.info(test_name + 'Assert actual status \'%s\' equals to expected status 201' % status)
_assert(self, status, 201, '')
self.mylog.info(test_name + 'STEP2: Verify org data was persisted in the etcd store')
verify_org_etcd_entries(self, test_name, created_org_id, created_org_name, error='', get_index_values=True,
id_by_index_name=created_org_id, error_by_index_name='')
self.mylog.info(test_name + 'STEP 3: Create Bucket \'%s\' name for org \'%s\' name'
% (bucket_name, org_name))
create_bucket_result = \
buckets_util.create_bucket(self, self.gateway, bucket_name, 3600, created_org_id)
status = create_bucket_result['status']
created_bucket_id = create_bucket_result['bucket_id']
self.mylog.info(test_name + 'Assert actual status \'%s\' equals to expected status 201' % status)
_assert(self, status, 201, '')
self.mylog.info(test_name + 'STEP 4: Verify bucket data was persisted in the etcd store')
verify_bucket_etcd_entries(self, test_name, created_bucket_id, bucket_name, expected_error='',
expected_retention_period=3600000000000)
self.footer(test_name)
def test_create_users_duplicate_names(self):
"""
REST API: http://<gateway>/api/v2/users
METHOD: POST
tests cannot create user with already existing name.
"""
test_name = 'test_create_users_duplicate_names '
user_name = 'duplicate_name'
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Create a user \'%s\'' % user_name)
status, created_user_id, created_user_name, error_message = \
gateway_util.create_user(self, self.gateway, user_name)
self.mylog.info(
test_name +
'Assert actual status code \'%s\' equals to expected status code \'%s\''
% (status, 201))
_assert(self, status, 201, 'status code')
self.mylog.info(test_name +
'STEP 2: Verify data was persisted in the etcd store')
verify_user_etcd_entries(self,
test_name,
created_user_id,
created_user_name,
expected_error='')
self.mylog.info(test_name +
'STEP 3: Try creating user with the same name')
status, created_user_id, created_user_name, error_message = \
gateway_util.create_user(self, self.gateway, user_name)
_assert(self,
status,
404,
'status code',
xfail=True,
reason='status code is \'%s\'' % status)
self.footer(test_name)
def test_create_duplicate_org(self):
"""
REST API: http://<gateway>/api/v2/orgs
METHOD: POST
tests cannot create org with already existing name.
"""
test_name = 'test_create_duplicate_org '
org_name = 'duporgname'
self.header(test_name)
self.mylog.info(test_name +
' STEP 1: Create Organization %s' % org_name)
create_result = org_util.create_organization(self, self.gateway,
org_name)
status = create_result['status']
created_org_id = create_result['org_id']
created_org_name = create_result['org_name']
_assert(self, status, 201, 'status code')
self.mylog.info(test_name +
'STEP 2: Verify data was persisted in the etcd store')
verify_org_etcd_entries(self, test_name, created_org_id,
created_org_name, '')
self.mylog.info(test_name + 'STEP 3: Creating org with the same name')
# TODO: According to @goller multiple organizations with the same name could be created,
# TODO: but with different ids, but currently it does not work
# TODO: filed a bug(gshif)
create_result = org_util.create_organization(self, self.gateway,
org_name)
status = create_result['status']
_assert(self,
status,
201,
'status code',
xfail=True,
reason='cannot create org with the same name')
self.footer(test_name)
def test_create_duplicate_bucket(self):
"""
REST API: http://<gateway>/api/v2/buckets
METHOD: POST
tests cannot create bucket with already existing name with the same org.
"""
test_name = 'test_create_duplicate_bucket '
org_name = 'orgname'
bucket_name = 'dupbucketname'
expected_error_message = 'bucket with name dupbucketname already exists'
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create Organization \'%s\'' % org_name)
create_org_result = org_util.create_organization(self, self.gateway, org_name)
status = create_org_result['status']
created_org_id = create_org_result['org_id']
created_org_name = create_org_result['org_name']
self.mylog.info(test_name + 'Assert actual status \'%s\' equals to expected status 201' % status)
_assert(self, status, 201, '')
self.mylog.info(test_name + 'STEP 2: Verify org data was persisted in the etcd store')
verify_org_etcd_entries(self, test_name, created_org_id, created_org_name, error='', get_index_values=True,
id_by_index_name=created_org_id, error_by_index_name='')
self.mylog.info(test_name + 'STEP 3: Create Bucket \'%s\'' % bucket_name)
create_bucket_result = \
buckets_util.create_bucket(self, self.gateway, bucket_name, 3600, created_org_id)
status = create_bucket_result['status']
created_bucket_id = create_bucket_result['bucket_id']
self.mylog.info(test_name + 'Assert actual status \'%s\' equals to expected status 201' % status)
_assert(self, status, 201, 'status code')
self.mylog.info(test_name + 'STEP 4: Verify bucket data was persisted in the etcd store')
verify_bucket_etcd_entries(self, test_name, created_bucket_id, bucket_name, expected_error='',
expected_retention_period=3600000000000)
self.mylog.info(test_name + 'STEP 5: Create Bucket with already existing name for the same org')
create_bucket_result = \
buckets_util.create_bucket(self, self.gateway, bucket_name, 3600, created_org_id)
error_message = create_bucket_result['error_message']
_assert(self, error_message, expected_error_message, 'error message')
self.footer(test_name)
def run_tests(self, name_of_the_test_to_run, org_name):
"""
:param name_of_the_test_to_run: test to be run
:param org_name: name of the organization to be created
:return: pass/fail
"""
test_name = name_of_the_test_to_run + org_name + ' '
self.header(test_name)
self.mylog.info(test_name +
' STEP 1: Create Organization "%s"' % org_name)
create_result = org_util.create_organization(self, self.gateway,
org_name)
status = create_result['status']
created_org_id = create_result['org_id']
created_org_name = create_result['org_name']
if org_name == '':
# TODO: According to @goller organization with empty name can be created. Currently there is a bug.
# TODO: Create a bug (@gshif)
_assert(self,
status,
201,
'status code',
xfail=True,
reason='Should be able to create an org with empty name')
elif org_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status code')
self.mylog.info(test_name +
'STEP 2: Verify data was persisted in the etcd store')
verify_org_etcd_entries(self,
test_name,
created_org_id,
created_org_name,
error='')
self.footer(test_name)
def test_create_bucket_default_rp(self):
"""
REST API: http://<gateway>/api/v2/buckets
METHOD: POST
tests bucket cannot be created if retention policy is not provided, (was before: default RP is used: 0s))
"""
test_name = 'test_create_bucket_default_rp'
org_name = 'org_default_rp'
bucket_name = 'bucket_default_rp'
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create Organization "%s"' % org_name)
create_org_result = org_util.create_organization(self, self.gateway, org_name)
status = create_org_result['status']
created_org_id = create_org_result['org_id']
created_org_name = create_org_result['org_name']
self.mylog.info(test_name + 'Assert actual status \'%s\' equals to expected status 201' % status)
_assert(self, status, 201, 'status code')
self.mylog.info(test_name + 'STEP 2: Verify org data was persisted in the etcd store')
verify_org_etcd_entries(self, test_name, created_org_id, created_org_name, error='', get_index_values=True,
id_by_index_name=created_org_id, error_by_index_name='')
self.mylog.info(test_name + 'STEP 3: Create Bucket \'%s\'' % bucket_name)
create_bucket_result = \
buckets_util.create_bucket(self, self.gateway, bucket_name, retention_rules=None,
organization_id=created_org_id)
status = create_bucket_result['status']
created_bucket_id = create_bucket_result['bucket_id']
retention_period = create_bucket_result['every_seconds']
self.mylog.info(test_name + 'Assert actual status \'%s\' equals to expected status 201' % status)
_assert(self, status, 201, 'status code')
self.mylog.info(test_name + 'Assert actual RP \'%s\' equals to expected RP \'%s\'' % (retention_period, 0))
_assert(self, retention_period, 0, 'retention policy')
self.mylog.info(test_name + 'STEP 4: Verify bucket data was persisted in the etcd store')
verify_bucket_etcd_entries(self, test_name, created_bucket_id, bucket_name, expected_error='',
expected_retention_period=0)
self.footer(test_name)
def run_tests(self, name_of_the_test_to_run, user_name):
"""
:param name_of_the_test_to_run: test name
:param user_name: name of the user to be created
:return: pass/fail
"""
test_name = name_of_the_test_to_run + user_name + ' '
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Create a user \'%s\'' % user_name)
status, created_user_id, created_user_name, error_message = \
gateway_util.create_user(self, self.gateway, user_name)
if user_name == '':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif user_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status code')
self.mylog.info(test_name +
'STEP 2: Verify data was persisted in the etcd store')
verify_user_etcd_entries(self,
test_name,
created_user_id,
created_user_name,
expected_error='')
self.footer(test_name)
def test_e2e_smoke(self):
"""
1. Create an organization. Verify organization was created successfully.
2. Create a bucket for the organization created in step 1. Verify org was created successfully.
3. Create a user. Verify user was created successfully.
4. Give user permissions to write data to buckets and read data from the same buckets. Verify authentication is
successful.
5. Write data points
5.1. Verify data got to kafka.
6. Query data using using queryd. (gateway:9999/api/v2/querysvc)
7. Query data using gateway. (gateway:9999/api/v2/query)
"""
test_name = 'test_e2e_smoke '
org_name = 'test_e2e_smoke_org'
bucket_name = 'test_e2e_smoke_bucket'
user_name = 'test_e2e_smoke_user'
measurement = 'test_m'
tag = 'hello world'
tag_storage = 'hello\ world'
value = '1234'
query = 'from(bucket:"%s") |> range(start:-5m)' % bucket_name
err = '' # intial error message is am empty string.
data = [
] # to store the data that made it to kafka, initial value is empty list.
final_kafka_points = '' # if kafka has a data we are looking for, added to a final_point str.
final_storage_points = ''
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Create Organization')
self.mylog.info(test_name + '\n')
create_org_result = org_util.create_organization(
self, self.gateway, org_name)
status_code = create_org_result.get('status')
org_id = create_org_result.get('org_id')
org_name = create_org_result.get('org_name')
_assert(self, status_code, 201, 'Create Organization Status Code')
verify_org_etcd_entries(self, test_name, org_id, org_name, error='')
self.mylog.info(test_name + 'STEP 2: Create Bucket')
self.mylog.info(test_name + '\n')
create_bucket_result = \
buckets_util.create_bucket(self, self.gateway, bucket_name, organization_id=org_id)
status_code = create_bucket_result['status']
created_bucket_id = create_bucket_result.get('bucket_id')
created_bucket_name = create_bucket_result.get('bucket_name')
_assert(self, status_code, 201, 'Create Bucket Status Code')
verify_bucket_etcd_entries(self,
test_name,
created_bucket_id,
created_bucket_name,
0,
expected_error='')
self.mylog.info(test_name + 'STEP 3: Create User')
self.mylog.info(test_name + '\n')
status_code, user_id, created_user_name, error_message = \
gateway_util.create_user(self, self.gateway, user_name)
_assert(self, status_code, 201, 'Create User Status Code')
permissions = [{
"action": "read",
"resource": "bucket/%s" % created_bucket_id
}, {
"action": "write",
"resource": "bucket/%s" % created_bucket_id
}]
self.mylog.info(
test_name +
'STEP 4: Create Authorization Token for \'%s\' to be able to read/write \'%s\' bucket'
% (user_name, bucket_name))
self.mylog.info(test_name + '\n')
create_auth_result = gateway_util.create_authorization(
self, self.gateway, user_name, user_id, json.dumps(permissions))
token = create_auth_result.get('token')
_assert(self, create_auth_result.get('status'), 201,
'Create Authorization Status')
_assert(self, create_auth_result.get('error'), None,
'Create Authorization Error')
self.mylog.info(test_name + 'STEP 5: Write point to a bucket')
self.mylog.info(test_name + '\n')
write_result = gateway_util.write_points(
self,
self.gateway,
token,
org_name,
bucket_name,
data='%s,t=%s f=%s' % (measurement, tag_storage, value))
_assert(self, write_result.get('status'), 204,
'Write Data Point To A Bucket')
_assert(self, write_result.get('error'), '',
'Write Data Error Message')
self.mylog.info(test_name + 'STEP 6: Verify Data Was Written To Kafka')
self.mylog.info(test_name + '\n')
# just if there is a latency on writing data to kafka, give it some time
end_time = time.time() + 10
while time.time() <= end_time:
topics, data, err = gateway_util.kafka_find_data_by_tag(
self, self.kubeconf, self.kubecluster, self.namespace, tag,
'kafka-0')
if len(data) == 0:
self.mylog.info(
test_name +
'KAFKA DOES NOT HAVE THE DATA YET. SLEEPING 1 SECOND.')
time.sleep(1)
continue
else:
for point in data:
if tag in point:
final_kafka_points += point
self.mylog.info(test_name +
'KAFKA DOES HAVE THE DATA:' +
str(point))
break
else:
self.mylog.info(test_name +
'KAFKA DOES NOT HAVE THE DATA YET')
continue
break
_assert(self, err, '', 'ERROR GETTING DATA FROM KAFKA')
_assert(self, len(data) > 0, True, 'KAFKA DOES NOT HAVE THE DATA')
_assert(self, tag in final_kafka_points, True,
'KAFKA DOES NOT HAVE THE DATA')
# storage is pulling every 10 seconds from kafka
self.mylog.info(test_name +
'STEP 7: Verify Data Was Written To Storage')
self.mylog.info(test_name + '\n')
end_time = time.time() + 20
while time.time() <= end_time:
engine, data, err = gateway_util.storage_find_data(
self, self.kubeconf, self.kubecluster, self.namespace, tag,
'storage-0')
if len(data) == 0:
self.mylog.info(
test_name +
'STORAGE DOES NOT HAVE THE DATA YET. SLEEPING FOR 1 SECOND.'
)
time.sleep(1)
continue
else:
for point in data:
if tag_storage in point:
final_storage_points += point
self.mylog.info(test_name +
'STORAGE DOES HAVE THE DATA: ' +
str(point))
break
else: # storage has some data, but data we are looking for is not there yet.
self.mylog.info('STORAGE DOES NOT HAVE THE DATA YET')
continue
break
_assert(self, err, '', 'ERROR GETTING DATA FROM STORAGE')
_assert(self, len(data) > 0, True, 'STORAGE DOES NOT HAVE THE DATA')
_assert(self, tag_storage in final_storage_points, True,
'STORAGE DOES NOT HAVE THE DATA')
self.mylog.info(test_name + 'STEP 8: Query Data using Queryd')
self.mylog.info(test_name + '\n')
# need to give it up to 30 sec to get the results back
end_time = time.time() + 30
result_queryd = None
while time.time() <= end_time:
result_queryd = gateway_util.queryd_query_data(self,
query,
self.flux,
org_id,
timeout=5,
responsenone=False)
if result_queryd.get('status') == 200 and len(
result_queryd.get('result')) == 1:
break
else:
self.mylog.info(
test_name +
'WAITING FOR QUERY RESULTS. SLEEPING FOR 1 SECOND.')
time.sleep(1)
_assert(self, result_queryd.get('status'), 200, ' STATUS CODE')
_assert(self, len(result_queryd.get('result')), 1,
' NUMBER OF RECORDS')
_assert(self,
result_queryd.get('result')[0].get('_measurement'),
measurement, 'Measurement')
_assert(self,
result_queryd.get('result')[0].get('_value'), value,
'Field Value')
_assert(self,
result_queryd.get('result')[0].get('t'), tag, 'Tag Value')
self.mylog.info('')
self.mylog.info(test_name + 'STEP 9: Query Data using Gateway')
self.mylog.info(test_name + '\n')
# need to give it up to 30 sec to get the results back
end_time = time.time() + 30
result_gateway = None
while time.time() <= end_time:
result_gateway = gateway_util.gateway_query_data(
self, query, self.gateway, token, org_name)
if result_gateway.get('status') == 200 and len(
result_gateway.get('result')) == 1:
break
else:
self.mylog.info(
test_name +
'WAITING FOR QUERY RESULTS. SLEEPING FOR 1 SECONDS.')
time.sleep(1)
_assert(self, result_gateway.get('status'), 200, ' STATUS CODE')
_assert(self, len(result_gateway.get('result')), 1,
' NUMBER OF RECORDS')
_assert(self,
result_gateway.get('result')[0].get('_measurement'),
measurement, 'Measurement')
_assert(self,
result_gateway.get('result')[0].get('_value'), value,
'Field Value')
_assert(self,
result_gateway.get('result')[0].get('t'), tag, 'Tag Value')
def run_tests(self, name_of_the_test_to_run, org_name):
"""
:param name_of_the_test_to_run: test to run
:param org_name:
:return: pass/fail
"""
test_name = name_of_the_test_to_run + org_name + ' '
org_name_to_assert = ''
if org_name == 'DoubleQuotes\\"':
org_name_to_assert = 'DoubleQuotes"_updated_name'
new_org_name = org_name + '_updated_name'
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create Organization')
create_result = org_util.create_organization(self, self.gateway,
org_name)
status = create_result['status']
created_org_id = create_result['org_id']
if org_name == '':
_assert(self,
status,
201,
'status_code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/188')
elif org_name == 'BackSlash\\':
_assert(self,
status,
201,
'status_code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status_code')
self.mylog.info(test_name +
' STEP 2: Update created org with the \'%s\' name' %
new_org_name)
update_result = org_util.update_organization(self, self.gateway,
created_org_id,
new_org_name)
status = update_result['status']
new_org_id = update_result['updated_org_id']
updated_org_name = update_result['updated_org_name']
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status 200' %
status)
_assert(self, status, 200, 'status_code')
self.mylog.info(
test_name +
'Assert actual org_id \'%s\' equals to expected org_id \'%s\'' %
(new_org_id, created_org_id))
_assert(self, new_org_id, created_org_id, 'org_id')
if org_name == 'DoubleQuotes\\"':
self.mylog.info(
test_name +
'Assert actual org_name \'%s\' equals to expected org_name \'%s\''
% (updated_org_name, org_name_to_assert))
_assert(self, updated_org_name, org_name_to_assert, 'org_name')
else:
self.mylog.info(
test_name +
'Assert actual org_name \'%s\' equals to expected org_name \'%s\''
% (updated_org_name, new_org_name))
_assert(self, updated_org_name, new_org_name, 'org_name')
self.mylog.info(
test_name +
'STEP 3: Verify updated name was persisted in the etcd store')
verify_org_etcd_entries(self,
test_name,
created_org_id=new_org_id,
created_org_name=updated_org_name,
error='')
self.footer(test_name)
def run_tests(self, name_of_the_test, user_name):
"""
:param name_of_the_test: name of the test to run
:param user_name: user name to get
:return: pass/fail
"""
test_name = name_of_the_test + user_name + ' '
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create User \'%s\'' % user_name)
status, created_user_id, created_user_name, error_message = \
gateway_util.create_user(self, self.gateway, user_name)
if user_name == '':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif user_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status code')
self.mylog.info(test_name +
'STEP 2: Verify data was persisted in the etcd store')
verify_user_etcd_entries(self,
test_name,
created_user_id,
created_user_name,
expected_error='')
self.mylog.info(test_name + ' STEP 3: Get Created User')
status, actual_user_id, actual_user_name, error_message = \
gateway_util.get_user_by_id(self, self.gateway, created_user_id)
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status \'%s\'' %
(status, 200))
_assert(self, status, 200, 'status code')
self.mylog.info(
test_name +
'Assert actual user_id \'%s\' equals expected user_id \'%s\'' %
(actual_user_id, created_user_id))
_assert(self, actual_user_id, created_user_id, 'user id')
self.mylog.info(
test_name +
'Assert actual user_name \'%s\' equals expected user_name \'%s\'' %
(actual_user_name, created_user_name))
_assert(self, actual_user_name, created_user_name, 'user name')
self.footer(test_name)
def run_tests(self, name_of_the_test_to_run, org_name, bucket_name, retention_period):
"""
:param name_of_the_test_to_run: test to be run
:param org_name: name of the org to be created
:param bucket_name: name of the bucket to be created
:param retention_period: retention period of the bucket. how long keep the data for. Default is 0s (forever)
:return: pass/fail
"""
test_name = name_of_the_test_to_run + org_name + ' '
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create Organization \'%s\'' % org_name)
self.mylog.info('')
create_org_result = org_util.create_organization(self, self.gateway, org_name)
status = create_org_result['status']
created_org_id = create_org_result['org_id']
created_org_name = create_org_result['org_name']
if org_name == '':
_assert(self, status, 201, 'status code', xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif org_name == 'BackSlash\\':
_assert(self, status, 201, 'status code', xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, '')
self.mylog.info(test_name + 'STEP 2: Verify org data was persisted in the etcd store')
verify_org_etcd_entries(self, test_name, created_org_id, created_org_name, error='', get_index_values=True,
id_by_index_name=created_org_id, error_by_index_name='')
self.mylog.info(test_name + 'STEP 3: Create Bucket "%s"' % bucket_name)
create_bucket_result = \
buckets_util.create_bucket(self, self.gateway, bucket_name, retention_period, created_org_id)
status = create_bucket_result['status']
created_bucket_id = create_bucket_result['bucket_id']
created_bucket_name = create_bucket_result['bucket_name']
if bucket_name == '':
_assert(self, status, 201, 'status code', xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif bucket_name == 'BackSlash\\':
_assert(self, status, 201, 'status code', xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status code')
self.mylog.info(test_name + 'STEP 4: Verify bucket data was persisted in the etcd store')
# since RP is stored in nanoseconds, need to convert PR that is passed to method (1h, 1m, 1s, etc) into int by
# removing the last character that indicates the duration in our case - h (hour)
# exp_retention_period = int(retention_period[:-1]) * 3600000000000
verify_bucket_etcd_entries(self, test_name, created_bucket_id, created_bucket_name, expected_error='',
expected_retention_period=3600000000000)
self.footer(test_name)
def run_tests(self,
name_of_the_test_to_run,
org_name,
bucket_name,
retention_period,
new_bucket_name=None,
new_retention=None):
"""
:param name_of_the_test_to_run: test to be run
:param org_name: name of the organization to be created
:param bucket_name: name of the bucket to be created
:param retention_period: retention period for the bucket
:param new_bucket_name: name of the bucket to update to
:param new_retention: retention period to update to
:return: pass/fail
"""
test_name = name_of_the_test_to_run + org_name + ' '
new_bucket_name_to_assert = ''
if org_name == 'DoubleQuotes\\"':
new_bucket_name_to_assert = 'DoubleQuotes"_updated'
self.header(test_name)
self.mylog.info(test_name +
' STEP 1: Create Organization \'%s\'' % org_name)
create_org_result = org_util.create_organization(
self, self.gateway, org_name)
status = create_org_result['status']
created_org_id = create_org_result['org_id']
created_org_name = create_org_result['org_name']
if org_name == '':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif org_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, '')
self.mylog.info(
test_name +
'STEP 2: Verify org data was persisted in the etcd store')
verify_org_etcd_entries(self,
test_name,
created_org_id,
created_org_name,
error='',
get_index_values=True,
id_by_index_name=created_org_id,
error_by_index_name='')
self.mylog.info(test_name + 'STEP 3: Create Bucket "%s"' % bucket_name)
create_bucket_result = \
buckets_util.create_bucket(self, self.gateway, bucket_name, retention_period, created_org_id)
status = create_bucket_result['status']
created_bucket_id = create_bucket_result['bucket_id']
created_bucket_name = create_bucket_result['bucket_name']
created_retention = create_bucket_result['every_seconds']
created_organization_id = create_bucket_result['org_id']
if bucket_name == '':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif bucket_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status code')
self.mylog.info(
test_name +
'STEP 4: Verify bucket data was persisted in the etcd store')
verify_bucket_etcd_entries(self,
test_name,
created_bucket_id,
created_bucket_name,
expected_error='',
expected_retention_period=3600000000000)
# Neither bucket name, nor the retention period are being updated.
if new_bucket_name is None and new_retention is None:
self.mylog.info(test_name +
'STEP 5: Bucket info is not getting updated')
update_bucket_result = \
buckets_util.update_bucket(self, self.gateway, created_bucket_id, created_bucket_name,
created_retention, new_bucket_name=None, new_retention=None)
updated_bucket_name = update_bucket_result['bucket_name']
retention = update_bucket_result['every_seconds']
status = update_bucket_result['status']
bucket_id = update_bucket_result['bucket_id']
organization_id = update_bucket_result['org_id']
self.mylog.info(
test_name +
'Assert updated bucket name \'%s\' equals to expected bucket name \'%s\''
% (updated_bucket_name, created_bucket_name))
_assert(self, updated_bucket_name, created_bucket_name,
'bucket name')
self.mylog.info(
test_name +
'Assert actual retention \'%s\' equals to expected \'%s\' retention'
% (retention, created_retention))
_assert(self, retention, created_retention, 'retention period')
# only bucket name is being updated
elif new_bucket_name is not None and new_retention is None:
self.mylog.info(test_name +
'STEP 5: Update created bucket with \'%s\' name' %
new_bucket_name)
update_bucket_result = \
buckets_util.update_bucket(self, self.gateway, created_bucket_id, created_bucket_name,
created_retention, new_bucket_name, new_retention=None)
updated_bucket_name = update_bucket_result['bucket_name']
retention = update_bucket_result['every_seconds']
status = update_bucket_result['status']
bucket_id = update_bucket_result['bucket_id']
organization_id = update_bucket_result['org_id']
if org_name == 'DoubleQuotes\\"':
self.mylog.info(
test_name +
'Assert updated bucket_name \'%s\' equals to expected bucket_name \'%s\''
% (updated_bucket_name, new_bucket_name_to_assert))
_assert(self, updated_bucket_name, new_bucket_name_to_assert,
'bucket name')
else:
self.mylog.info(
test_name +
'Assert updated bucket_name \'%s\' equals to expected bucket_name \'%s\''
% (updated_bucket_name, new_bucket_name))
_assert(self, updated_bucket_name, new_bucket_name, '')
self.mylog.info(
test_name +
'Assert updated retention \'%s\' equals to expected retention \'%s\''
% (retention, created_retention))
_assert(self, retention, created_retention, 'retention period')
# only retention is being updated
elif new_bucket_name is None and new_retention is not None:
self.mylog.info(
test_name +
'STEP 5: Update created bucket with \'%d\' retention name' %
new_retention)
update_bucket_result = \
buckets_util.update_bucket(self, self.gateway, created_bucket_id, created_bucket_name,
created_retention, new_bucket_name=None, new_retention=new_retention)
updated_bucket_name = update_bucket_result['bucket_name']
retention = update_bucket_result['every_seconds']
status = update_bucket_result['status']
bucket_id = update_bucket_result['bucket_id']
organization_id = update_bucket_result['org_id']
self.mylog.info(
test_name +
'Assert updated bucket_name \'%s\' equals to expected bucket_name \'%s\''
% (updated_bucket_name, created_bucket_name))
_assert(self, updated_bucket_name, created_bucket_name,
'bucket name')
self.mylog.info(
test_name +
'Assert updated retention \'%s\' equals to expected retention \'%s\''
% (retention, new_retention))
_assert(self, retention, new_retention, 'retention period')
# bucket name and retention period are being updated
else:
self.mylog.info(
test_name +
'STEP 5: Update created bucket with \'%s\' bucket name and \'%d\' retention'
% (new_bucket_name, new_retention))
update_bucket_result = \
buckets_util.update_bucket(self, self.gateway, created_bucket_id, created_bucket_name,
created_retention, new_bucket_name, new_retention)
retention = update_bucket_result['every_seconds']
updated_bucket_name = update_bucket_result['bucket_name']
status = update_bucket_result['status']
bucket_id = update_bucket_result['bucket_id']
organization_id = update_bucket_result['org_id']
self.mylog.info(
test_name +
'Assert updated retention \'%s\' equals to expected retention \'%s\''
% (retention, new_retention))
_assert(self, retention, new_retention, 'retention period')
if org_name == 'DoubleQuotes\\"':
self.mylog.info(
test_name +
'Assert updated bucket_name \'%s\' equals to expected bucket_name \'%s\''
% (updated_bucket_name, new_bucket_name_to_assert))
_assert(self, updated_bucket_name, new_bucket_name_to_assert,
'bucket name')
else:
self.mylog.info(
test_name +
'Assert updated bucket_name \'%s\' equals to expected bucket_name \'%s\''
% (updated_bucket_name, new_bucket_name))
_assert(self, updated_bucket_name, new_bucket_name, '')
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status \'%s\'' %
(status, 200))
_assert(self, status, 200, 'status code')
self.mylog.info(
test_name +
'Assert actual bucket_id \'%s\' equals to expected bucket_id \'%s\''
% (bucket_id, created_bucket_id))
_assert(self, bucket_id, created_bucket_id, 'bucket id')
self.mylog.info(
test_name +
'Assert actual org_id \'%s\' equals to expected org_id \'%s\'' %
(organization_id, created_organization_id))
_assert(self, organization_id, created_organization_id,
'organization id')
self.mylog.info('')
self.mylog.info(
test_name +
'STEP 6: Verify updated bucket info was persisted in the etcd store'
)
verify_bucket_etcd_entries(self,
test_name,
bucket_id,
updated_bucket_name,
expected_error='',
expected_retention_period=3600000000000)
self.footer(test_name)
def run_tests(self, name_of_the_test_to_run, user_name):
"""
:param name_of_the_test_to_run: name of the test to be run
:param user_name: name of user under the test
:return: pass/fail
"""
test_name = name_of_the_test_to_run + user_name + ' '
new_name_to_assert = ''
if user_name == 'DoubleQuotes\\"':
new_name_to_assert = 'DoubleQuotes"_updated_name'
new_name = user_name + '_updated_name'
self.header(test_name)
self.mylog.info(test_name + ' STEP 1: Create User \'%s\'' % user_name)
status, created_user_id, created_user_name, error_message = \
gateway_util.create_user(self, self.gateway, user_name)
if user_name == '':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif user_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status code')
self.mylog.info(test_name +
'STEP 2: Verify data was persisted in the etcd store')
verify_user_etcd_entries(self,
test_name,
created_user_id,
created_user_name,
expected_error='')
self.mylog.info(test_name +
' STEP 3: Update created user with the \'%s\' name' %
new_name)
status, new_user_id, updated_user_name, error_message = \
gateway_util.update_user(self, self.gateway, created_user_id, new_name)
self.mylog.info(
test_name +
'Assert actual status \'%s\' equals to expected status \'%s\'' %
(status, 200))
_assert(self, status, 200, 'status code')
self.mylog.info(
test_name +
'Assert updated user_id \'%s\' equals to expected user_id \'%s\'' %
(new_user_id, created_user_id))
_assert(self, new_user_id, created_user_id, 'user id')
if user_name == 'DoubleQuotes\\"':
self.mylog.info(
test_name +
'Assert updated user_name \'%s\' equals to expected user_name \'%s\''
% (updated_user_name, new_name_to_assert))
_assert(self, updated_user_name, new_name_to_assert, 'user name')
else:
self.mylog.info(
test_name +
'Assert updated user_name \'%s\' equals to expected user_name \'%s\''
% (updated_user_name, new_name))
_assert(self, updated_user_name, new_name, 'user name')
self.mylog.info(
test_name +
'STEP 4: Verify updated name was persisted in the etcd store')
verify_user_etcd_entries(self,
test_name,
new_user_id,
updated_user_name,
expected_error='')
self.footer(test_name)
def test_tasks_smoke(self):
"""
1. Create organization and verify it was created successfully.
2. Create bucket (source bucket) to read data from and verify it was created successfully.
3. Create bucket (destination bucket) to write data to and verify it was created successfully.
4. Create user.
5. Create an authorization token.
6. Create task.
7. Verify created task.
8. Write a point to a bucket the data is read from.
9. Verify data was written to kafka.
10. Verify data was written to storage.
11. Query written data using gateway endpoint.
12. Wait for task to be executed.
13. Verify data was written to kafka.
14. Verify data was written to a bucket we wrote a data to.
15. Query the data from a bucket the data was written to.
"""
test_name = 'test_tasks_smoke '
org_name = 'test_tasks_smoke_org'
bucket_name_destination = 'test_tasks_smoke_bucket_dest'
bucket_name_source = 'test_tasks_smoke_bucket_source'
user_name = 'test_tasks_smoke_user'
task_description = 'task_description'
task_status = 'active'
task_name = 'test_task_smoke_name'
task_duration = '30s'
data = [
] # to store the data that made it to kafka, initial value is empty list
err = '' # initially there is no error
task_tag = 'taskcreation'
measurement = 'task_m'
value = '1234'
tag = 'qa_great'
result_gateway = {}
final_kafka_points = ''
final_storage_points = ''
self.header(test_name)
self.mylog.info(test_name + 'STEP 1: Create Organization')
self.mylog.info(test_name + '\n')
create_org = org_util.create_organization(self, self.gateway, org_name)
status = create_org.get('status')
_assert(self, status, 201, 'RESPONSE STATUS')
org_id = create_org.get('org_id')
org_name = create_org.get('org_name')
verify_org_etcd_entries(self, test_name, org_id, org_name, error='')
# create bucket to read data from - source bucket
self.mylog.info(test_name + 'STEP 2: Create Bucket To Read Data From')
self.mylog.info(test_name + '\n')
create_bucket_source = buckets_util.create_bucket(
self, self.gateway, bucket_name_source, organization_id=org_id)
status = create_bucket_source.get('status')
_assert(self, status, 201, 'RESPONSE STATUS')
bucket_id_created_source = create_bucket_source.get('bucket_id')
bucket_name_created_source = create_bucket_source.get('bucket_name')
verify_bucket_etcd_entries(self,
test_name,
bucket_id_created_source,
bucket_name_created_source,
0,
expected_error='')
# create bucket to write data to
self.mylog.info(test_name + 'STEP 3: Create Bucket To Write Data To')
self.mylog.info(test_name + '\n')
create_bucket_dest = buckets_util.create_bucket(
self,
self.gateway,
bucket_name_destination,
organization_id=org_id)
status = create_bucket_dest.get('status')
_assert(self, status, 201, 'RESPONSE STATUS')
bucket_id_created_dest = create_bucket_dest.get('bucket_id')
bucket_name_created_dest = create_bucket_dest.get('bucket_name')
verify_bucket_etcd_entries(self,
test_name,
bucket_id_created_dest,
bucket_name_created_dest,
0,
expected_error='')
# create user
self.mylog.info(test_name + 'STEP 4: Create User')
self.mylog.info(test_name + '\n')
create_user = gateway_util.create_user(self, self.gateway, user_name)
status = create_user[0]
_assert(self, status, 201, 'RESPONSE STATUS')
user_id = create_user[1]
user_name = create_user[2]
verify_user_etcd_entries(self,
test_name,
user_id,
user_name,
expected_error='')
# give user permissions
self.mylog.info(
test_name +
'STEP 5: Create Authorization Token For Tasks/Write And Read Operations'
)
self.mylog.info(test_name + '\n')
tasks_permissions = [{
"action": "create",
"resource": "org/%s/task" % org_id
}, {
"action": "read",
"resource": "bucket/%s" % bucket_id_created_source
}, {
"action": "write",
"resource": "bucket/%s" % bucket_id_created_dest
}]
write_permissions = [{
"action": "write",
"resource": "bucket/%s" % bucket_id_created_source
}]
read_permissions = [{
"action": "write",
"resource": "bucket/%s" % bucket_id_created_dest
}, {
"action": "read",
"resource": "bucket/%s" % bucket_id_created_source
}]
create_tasks_permissions = \
gateway_util.create_authorization(self, self.gateway, user_name, user_id, json.dumps(tasks_permissions))
status = create_tasks_permissions.get('status')
_assert(self, status, 201, 'RESPONSE STATUS')
tasks_token = create_tasks_permissions.get('token')
create_write_permissions = \
gateway_util.create_authorization(self, self.gateway, user_name, user_id, json.dumps(write_permissions))
status = create_write_permissions.get('status')
_assert(self, status, 201, 'RESPONSE STATUS')
write_token = create_write_permissions.get('token')
create_read_permissions = \
gateway_util.create_authorization(self, self.gateway, user_name, user_id, json.dumps(read_permissions))
status = create_read_permissions.get('status')
_assert(self, status, 201, 'RESPONSE STATUS')
read_token = create_read_permissions.get('token')
self.mylog.info(test_name + 'STEP 6: Create Task')
self.mylog.info(test_name + '\n')
flux_script = 'option task = {name:"%s", every:%s} from(bucket:"%s") |> range(start: -1h) ' \
'|> map(fn: (r) => ({_time: r._time, _value:r._value, t : "%s"}))' \
'|> to(bucket:"%s", orgID:"%s")' % \
(task_name, task_duration, bucket_name_source, task_tag, bucket_name_destination, org_id)
create_task = tasks_util.create_task(self,
self.gateway,
org_id,
task_description,
tasks_token,
task_status,
flux=flux_script)
time_task_created = time.time()
self.mylog.info(test_name + 'TASK WAS CREATED AT : ' +
str(datetime.now()))
status = create_task.get('status')
_assert(self, status, 201, 'RESPONSE STATUS')
task_id = create_task.get('task_id')
# query etcd-tasks
etcd_tasks = gateway_util.get_tasks_etcd(self, self.etcd_tasks,
task_id)
self.mylog.info(test_name + 'ETCD TASKS RESULT : ' + str(etcd_tasks))
self.mylog.info(test_name + 'STEP 7: Verify Created Task')
self.mylog.info(test_name + '\n')
_assert(self, etcd_tasks.get('flux_script'), flux_script,
'FLUX SCRIPT')
_assert(self, etcd_tasks.get('task_name'), task_name, 'TASK_NAME')
_assert(self, etcd_tasks.get('org_id'), org_id, 'ORG ID')
_assert(self, etcd_tasks.get('user_id'), user_id, 'USER ID')
_assert(self, etcd_tasks.get('status'), task_status, 'TASK STATUS')
_assert(self, etcd_tasks.get('schedule'), 'every %s' % task_duration,
'TASK SCHEDULE')
# write a point
self.mylog.info(test_name + 'STEP 8: Write a point')
self.mylog.info(test_name + '\n')
write_result = gateway_util.write_points(self,
self.gateway,
write_token,
org_name,
bucket_name_source,
data='%s,t=%s f=%s' %
(measurement, tag, value))
_assert(self, write_result.get('status'), 204,
'Write Data Point To A Bucket')
_assert(self, write_result.get('error'), '',
'Write Data Error Message')
self.mylog.info(test_name + 'STEP 9: Verify Data Was Written To Kafka')
self.mylog.info(test_name + '\n')
end_time = time.time(
) + 10 # data should be written to kafka right away, but give it up to 10 sec
while time.time() <= end_time:
topics, data, err = gateway_util.kafka_find_data_by_tag(
self, self.kubeconf, self.kubecluster, self.namespace, tag,
'kafka-0')
if len(
data
) == 0: # kafka does not have any data, usually new deployments
self.mylog.info(
test_name +
'KAFKA DOES NOT HAVE THE DATA YET. SLEEPING FOR 1 SECOND')
time.sleep(1)
continue
else:
for point in data:
if tag in point:
final_kafka_points += point
self.mylog.info(test_name +
'KAFKA DOES HAVE THE DATA: ' +
str(point))
break
else: # there is already data in kafka, but the data we are looking for is not there yet.
self.mylog.info('KAFKA DOES NOT HAVE THE DATA YET')
continue
break
_assert(self, err, '', 'ERROR GETTING DATA FROM KAFKA')
_assert(self, len(data) > 0, True, 'KAFKA DOES NOT HAVE THE DATA')
_assert(self, tag in final_kafka_points, True,
'KAFKA DOES NOT HAVE THE DATA')
# storage is pulling every 10 seconds from kafka
self.mylog.info(test_name +
'STEP 10: Verify Data Was Written To Storage')
end_time = time.time() + 20
while time.time() <= end_time:
engine, data, err = gateway_util.storage_find_data(
self, self.kubeconf, self.kubecluster, self.namespace, tag,
'storage-0')
if len(
data
) == 0: # storage does not have any data yet, new deployments
self.mylog.info(
test_name +
'STORAGE DOES NOT HAVE THE DATA YET. SLEEPING FOR 1 SECOND'
)
time.sleep(1)
continue
else:
for point in data:
if tag in point:
final_storage_points += point
self.mylog.info(test_name +
'STORAGE DOES HAVE THE DATA: ' +
str(point))
break
else: # storage has some data, but data we are looking for is not there yet.
self.mylog.info('STORAGE DOES NOT HAVE THE DATA YET')
continue
break
_assert(self, err, '', 'ERROR GETTING DATA FROM STORAGE')
_assert(self, len(data) > 0, True, 'STORAGE DOES NOT HAVE THE DATA')
_assert(self, tag in final_storage_points, True,
'STORAGE DOES NOT HAVE THE DATA')
# query the data from a bucket
self.mylog.info(test_name + 'STEP 11: Query Data using Gateway')
self.mylog.info(test_name + '\n')
query_out = 'from(bucket:"%s") |> range(start:-5m)' % bucket_name_source
end_time = time.time(
) + 15 # should happen right away, since data is already in storage
while time.time() <= end_time:
result_gateway = gateway_util.gateway_query_data(
self, query_out, self.gateway, read_token, org_name)
if result_gateway.get('status') == 200 and len(
result_gateway.get('result')) == 1:
break
else:
self.mylog.info(test_name +
'WAITING FOR QUERY RESULTS. SLEEPING 1 SECOND')
time.sleep(1)
_assert(self, result_gateway.get('status'), 200, ' STATUS CODE')
_assert(self, len(result_gateway.get('result')), 1,
' NUMBER OF RECORDS')
_assert(self,
result_gateway.get('result')[0].get('_measurement'),
measurement, 'Measurement')
_assert(self,
result_gateway.get('result')[0].get('_value'), value,
'Field Value')
_assert(self,
result_gateway.get('result')[0].get('t'), tag, 'Tag Value')
# Wait for task service to execute the flux query
query_in = 'from(bucket:"%s") |> range(start:-5m)' % bucket_name_destination
# task wil start running at the end of the 'end_time', which is time task was created + 40 sec, since task will
# start running for the first time after its scheduled time passes, that is 30sec.
self.mylog.info(test_name + 'STEP 12: Wait For Task To Be Executed')
self.mylog.info(test_name + '\n')
end_time = time_task_created + 40
while time.time() < end_time:
time.sleep(1)
self.mylog.info(test_name +
'STEP 13: Verify Data Was Written To Kafka')
self.mylog.info(test_name + '\n')
final_kafka_points = ''
end_time = time.time() + 10
while time.time() <= end_time:
topics, data, err = gateway_util.kafka_find_data_by_tag(
self, self.kubeconf, self.kubecluster, self.namespace,
task_tag, 'kafka-0')
if len(data) == 0:
self.mylog.info(
test_name +
'KAFKA DOES NOT HAVE THE DATA YET. SLEEPING 1 SECOND.')
time.sleep(1)
continue
else:
for point in data:
if task_tag in point and not 'task-system' in point:
final_kafka_points += point
self.mylog.info(test_name +
'KAFKA DOES HAVE THE DATA:' +
str(point))
break
else:
self.mylog.info(test_name +
'KAFKA DOES NOT HAVE THE DATA YET')
continue
break
_assert(self, err, '', 'ERROR GETTING DATA FROM KAFKA')
_assert(self, len(data) > 0, True, 'KAFKA DOES NOT HAVE THE DATA')
_assert(self, task_tag in final_kafka_points, True,
'KAFKA DOES NOT HAVE THE DATA')
# storage is pulling every 10 seconds from kafka
self.mylog.info(test_name +
'STEP 14: Verify Data Was Written To Storage')
self.mylog.info(test_name + '\n')
final_storage_points = ''
end_time = time.time() + 20
while time.time() <= end_time:
engine, data, err = gateway_util.storage_find_data(
self, self.kubeconf, self.kubecluster, self.namespace,
task_tag, 'storage-0')
if len(data) == 0:
self.mylog.info(
test_name +
'STORAGE DOES NOT HAVE THE DATA YET. SLEEPING 1 SECOND')
time.sleep(1)
continue
else:
for point in data:
if task_tag in point and not 'task-system' in point:
final_storage_points += point
self.mylog.info(test_name +
'STORAGE DOES HAVE THE DATA:' +
str(point))
break
else:
self.mylog.info(test_name +
'STORAGE DOES NOT HAVE THE DATA YET')
continue
break
_assert(self, err, '', 'ERROR GETTING DATA FROM STORAGE')
_assert(self, len(data) > 0, True, 'STORAGE DOES NOT HAVE THE DATA')
_assert(self, task_tag in final_storage_points, True,
'STORAGE DOES NOT HAVE THE DATA')
self.mylog.info(test_name + 'STEP 15: Query Data Using Gateway')
self.mylog.info(test_name + '\n')
end_time = time.time() + 30
while time.time() <= end_time:
result_gateway = gateway_util.gateway_query_data(
self, query_in, self.gateway, read_token, org_name)
if result_gateway.get('status') == 200 and len(
result_gateway.get('result')) == 1:
break
else:
self.mylog.info(test_name +
'WAITING FOR QUERY RESULTS. SLEEPING 2 SECOND')
time.sleep(2)
_assert(self, result_gateway.get('status'), 200, ' STATUS CODE')
_assert(self, len(result_gateway.get('result')), 1,
' NUMBER OF RECORDS')
_assert(self,
result_gateway.get('result')[0].get('_measurement'),
measurement, 'Measurement')
_assert(self,
result_gateway.get('result')[0].get('_value'), value,
'Field Value')
_assert(self,
result_gateway.get('result')[0].get('t'), task_tag,
'Tag Value')
def run_tests(self, name_of_the_test_to_run, org_name):
"""
:param name_of_the_test_to_run: test name to run
:param org_name: name of the organization
:return: pass/fail
"""
test_name = name_of_the_test_to_run + org_name + ' '
self.header(test_name)
self.mylog.info(test_name +
' STEP 1: Create Organization "%s"' % org_name)
create_result = org_util.create_organization(self, self.gateway,
org_name)
status = create_result['status']
created_org_id = create_result['org_id']
created_org_name = create_result['org_name']
if org_name == '':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/188')
elif org_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status code')
self.mylog.info(
test_name +
' STEP 2: Check the organization data was persisted in etcd')
verify_org_etcd_entries(self,
test_name,
created_org_id,
created_org_name,
error='')
self.mylog.info(test_name + ' STEP 3: Get Created Organization')
get_result = org_util.get_organization_by_id(self, self.gateway,
created_org_id)
status = get_result['status']
actual_org_id = get_result['requested_org_id']
actual_org_name = get_result['requested_org_name']
self.mylog.info(
test_name +
'Assert actual status code \'%s\' equals expected status code 200')
_assert(self, status, 200, 'status code')
self.mylog.info(
test_name +
'Assert actual org_id \'%s\' equals to expected org_id \'%s\'' %
(actual_org_id, created_org_id))
_assert(self, actual_org_id, created_org_id, 'org id')
self.mylog.info(
test_name +
'Assert actual org_name \'%s\' equals expected org_name \'%s\'' %
(actual_org_name, created_org_name))
_assert(self, actual_org_name, created_org_name, 'org name')
self.footer(test_name)
def run_tests(self, name_of_the_test_to_run, org_name, bucket_name,
retention_period):
"""
:param name_of_the_test_to_run: test to be run
:param org_name: name of the organization to be created
:param bucket_name: name of the bucket to be created
:param retention_period: retention period of the bucket
:return: pass/fail
"""
test_name = name_of_the_test_to_run + org_name + ' '
self.header(test_name)
self.mylog.info(test_name +
' STEP 1: Create Organization "%s"' % org_name)
self.mylog.info(test_name + '')
create_org_result = org_util.create_organization(
self, self.gateway, org_name)
status = create_org_result['status']
created_org_name = create_org_result['org_name']
created_org_id = create_org_result['org_id']
if org_name == '':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif org_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, '')
self.mylog.info(
test_name +
'STEP 2: Verify org data was persisted in the etcd store')
self.mylog.info(test_name + '')
verify_org_etcd_entries(self,
test_name,
created_org_id,
created_org_name,
error='',
get_index_values=True,
id_by_index_name=created_org_id,
error_by_index_name='')
self.mylog.info(test_name + 'STEP 3: Create Bucket "%s"' % bucket_name)
self.mylog.info(test_name + '')
create_bucket_result = \
buckets_util.create_bucket(self, self.gateway, bucket_name, retention_period, created_org_id)
status = create_bucket_result['status']
created_bucket_id = create_bucket_result['bucket_id']
created_bucket_name = create_bucket_result['bucket_name']
if bucket_name == '':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/162')
elif bucket_name == 'BackSlash\\':
_assert(self,
status,
201,
'status code',
xfail=True,
reason='https://github.com/influxdata/platform/issues/163')
else:
_assert(self, status, 201, 'status code')
self.mylog.info(
test_name +
'STEP 4: Verify bucket data was persisted in the etcd store')
self.mylog.info(test_name + '')
# since RP is stored in nanoseconds, need to convert PR that is passed to method in seconds.
exp_retention_period = int(retention_period) * 1000000000
verify_bucket_etcd_entries(
self,
test_name,
created_bucket_id,
created_bucket_name,
expected_error='',
expected_retention_period=exp_retention_period)
self.mylog.info(test_name + ' STEP 5: Get Created Bucket')
self.mylog.info(test_name + '')
get_buckets_result = buckets_util.get_bucket_by_id(
self, self.gateway, created_bucket_id)
status = get_buckets_result['status']
actual_bucket_name = get_buckets_result['bucket_name']
actual_bucket_id = get_buckets_result['bucket_id']
self.mylog.info(
test_name +
'Assert actual status \'%s\'equals to expected status \'%s\'' %
(status, 200))
_assert(self, status, 200, '')
self.mylog.info(
test_name +
'Assert actual bucket_id \'%s\' equals to expected bucket id \'%s\''
% (actual_bucket_name, created_bucket_id))
_assert(self, actual_bucket_id, created_bucket_id, '')
self.mylog.info(
test_name +
'Assert actual bucket_name \'%s\' equals to expected bucket_name \'%s\''
% (actual_bucket_name, created_bucket_name))
_assert(self, actual_bucket_name, created_bucket_name, '')
self.footer(test_name)
