def create_test_data_sets(cls, request, test_org, class_context,
test_data_urls):
step("Create new transfer for each category")
cls.transfers = []
for category in DataSet.CATEGORIES:
cls.transfers.append(
Transfer.api_create(class_context,
category,
is_public=False,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url))
for category in DataSet.CATEGORIES:
cls.transfers.append(
Transfer.api_create(class_context,
category,
is_public=True,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url))
step("Ensure that transfers are finished")
for transfer in cls.transfers:
transfer.ensure_finished()
step("Get all data sets in the test org")
cls.transfer_titles = [t.title for t in cls.transfers]
dataset_list = DataSet.api_get_list(org_guid_list=[test_org.guid])
cls.datasets = [
d for d in dataset_list if d.title in cls.transfer_titles
]
def test_add_and_delete_transfer_from_link(core_org, context, test_data_urls):
"""
<b>Description:</b>
Checks if a file can be downloaded from a link to the platform.
<b>Input data:</b>
1. File URL.
2. Transfer title.
3. Transfer category.
<b>Expected results:</b>
Test passes when a file from link was downloaded to the platform and corresponding transfer and dataset were
successfully created and removed from the platform.
<b>Steps:</b>
1. Create a transfer using csv file.
2. Check if transfer finished with correct state and verify that it is present on the transfers list.
3. Check if the transfer has corresponding dataset.
4. Delete dataset and transfer from the platform.
5. Verify that transfer and dataset were removed from the platform.
"""
step("Create a transfer")
transfer = Transfer.api_create(context,
category="other",
source=test_data_urls.test_transfer.url,
org_guid=core_org.guid)
transfer_flow(transfer, core_org)
def test_admin_can_get_transfer_list(self, test_org):
"""
<b>Description:</b>
Check that admin can get a transfers list.
<b>Input data:</b>
1. organization id
<b>Expected results:</b>
Test passes when transfer list is retrieved.
<b>Steps:</b>
1. Retrieve transfer list from organization as admin.
"""
step("Check if the list of transfers can be retrieved")
Transfer.api_get_list(org_guid_list=[test_org.guid])
def test_create_transfer_with_new_category(self, context, test_org):
"""
<b>Description:</b>
Check that transfer can be created from an url with custom category.
<b>Input data:</b>
1. organization id
2. custom category name
3. url with a source file
<b>Expected results:</b>
Transfer is successfully created with custom category.
<b>Steps:</b>
1. Create transfer with custom category.
2. Retrieve transfer from list.
3. Check that retrieved transfer has a custom category.
"""
new_category = "user_category"
transfer = self._create_transfer(context,
category=new_category,
org_guid=test_org.guid)
step("Get transfer and check it's category")
retrieved_transfer = Transfer.api_get(transfer.id)
assert retrieved_transfer.category == new_category, "Created transfer has different category"
def test_add_and_delete_transfer_from_file(core_org, context):
"""
<b>Description:</b>
Checks if a file can be uploaded from disk to the platform.
<b>Input data:</b>
1. File name.
2. Transfer title.
3. Transfer category.
<b>Expected results:</b>
Test passes when a file from disk was uploaded to the platform and corresponding transfer and dataset were
successfully created and removed from the platform.
<b>Steps:</b>
1. Create a transfer using generated csv file.
2. Check if transfer finished with correct state and verify that it is present on the transfers list.
3. Check if the transfer has corresponding dataset.
4. Delete dataset and transfer from the platform.
5. Verify that transfer and dataset were removed from the platform.
"""
step("Generate a test csv file")
file_path = generate_csv_file(column_count=10, row_count=100)
step("Create a transfer by file upload")
transfer = Transfer.api_create_by_file_upload(context,
category="health",
org_guid=core_org.guid,
file_path=file_path)
transfer_flow(transfer, core_org)
def test_add_and_delete_transfer_from_link(core_org, context):
"""Create and Delete Transfer from Link"""
step("Create a transfer")
transfer = Transfer.api_create(context,
category="other",
source=Urls.test_transfer_link,
org_guid=core_org.guid)
transfer_flow(transfer, core_org)
def _create_transfer(self, context, org_guid, category):
step("Create new transfer and wait until it's finished")
transfer = Transfer.api_create(context,
category=category,
source=Urls.test_transfer_link,
org_guid=org_guid)
transfer.ensure_finished()
return transfer
def check_transfer_and_dataset_are_visible_in_test_org(
transfer, dataset, test_org):
step("Check transfer is visible on list of transfers")
transfers = Transfer.api_get_list(org_guid_list=[test_org.guid])
assert transfer in transfers
step("Check dataset is visible on list of datasets")
datasets = DataSet.api_get_list(org_guid_list=[test_org.guid])
assert dataset in datasets
def test_create_transfer_with_new_category(self, context, test_org):
new_category = "user_category"
transfer = self._create_transfer(context,
category=new_category,
org_guid=test_org.guid)
step("Get transfer and check it's category")
retrieved_transfer = Transfer.api_get(transfer.id)
assert retrieved_transfer.category == new_category, "Created transfer has different category"
def test_create_transfer_without_category(self, context, test_org):
transfer = self._create_transfer(context,
category=None,
org_guid=test_org.guid)
step("Get transfer and check it's category")
transfer_list = Transfer.api_get_list(org_guid_list=[test_org.guid])
transfer.category = "other"
assert transfer in transfer_list, "Transfer was not found"
def test_3_create_data_set_and_publish_it_in_hive(self, class_context, test_org):
step("Create transfer and check it's finished")
transfer = Transfer.api_create(class_context, source=Urls.test_transfer_link, org_guid=test_org.guid,
title=self.transfer_title)
transfer.ensure_finished()
step("Publish in hive the data set created based on the submitted transfer")
data_set = DataSet.api_get_matching_to_transfer(org=test_org, transfer_title=self.transfer_title)
data_set.api_publish()
self.__class__.data_set_hdfs_path = ATKtools.dataset_uri_to_atk_uri(data_set.target_uri)
def dataset_target_uri(self, test_org, test_space, class_context, admin_user):
step("Add admin to space with developer role")
admin_user.api_add_to_space(test_space.guid, test_org.guid, User.SPACE_ROLES["developer"])
step("Create transfer")
transfer = Transfer.api_create(class_context, org_guid=test_org.guid, source=Urls.test_transfer_link)
transfer.ensure_finished()
step("Get dataset")
dataset = DataSet.api_get_matching_to_transfer(transfer.title, test_org)
return dataset.target_uri
def test_submit_transfer(self, context, test_org):
transfer = self._create_transfer(context,
category=self.DEFAULT_CATEGORY,
org_guid=test_org.guid)
step(
"Get transfers and check if they are the same as the uploaded ones"
)
retrieved_transfer = Transfer.api_get(transfer.id)
assert transfer == retrieved_transfer, "The transfer is not the same"
def test_add_and_delete_transfer_from_file(core_org, context):
"""Create and Delete Transfer from File"""
step("Generate a test csv file")
file_path = generate_csv_file(column_count=10, row_count=100)
step("Create a transfer by file upload")
transfer = Transfer.api_create_by_file_upload(context,
category="health",
org_guid=core_org.guid,
file_path=file_path)
transfer_flow(transfer, core_org)
def test_submit_and_delete_transfer(self, context, test_org):
"""
<b>Description:</b>
Check transfer creation from an url and deletion. Also check dataset deletion.
<b>Input data:</b>
1. organization guid
2. transfer category
3. url with a source file
<b>Expected results:</b>
Transfer is successfully created and deleted. Dataset is successfully deleted.
<b>Steps:</b>
1. Create transfer.
2. Retrieve corresponding dataset,
3. Delete transfer.
4. Check that transfer disappeared from the transfer list.
5. Delete dataset.
6. Check that dataset disappeared from the dataset list.
"""
transfer = self._create_transfer(context,
category=self.DEFAULT_CATEGORY,
org_guid=test_org.guid)
step(
"Get transfers and check if they are the same as the uploaded ones"
)
retrieved_transfer = Transfer.api_get(transfer.id)
assert transfer == retrieved_transfer, "The transfer is not the same"
dataset = DataSet.api_get_matching_to_transfer(
org_guid=test_org.guid, transfer_title=transfer.title)
TestSubmitTransfer.check_transfer_and_dataset_are_visible_in_test_org(
transfer=transfer, dataset=dataset, test_org=test_org)
step("Delete transfer")
transfer.cleanup()
step("Check transfer is not visible on list of transfers")
transfers = Transfer.api_get_list(org_guid_list=[test_org.guid])
assert transfer not in transfers
step("Delete dataset")
dataset.cleanup()
step("Check dataset is not visible on list of datasets")
datasets = DataSet.api_get_list(org_guid_list=[test_org.guid])
assert dataset not in datasets
def _get_transfer_and_dataset(self, test_org, file_source, access,
context):
step("Create transfer by providing a csv from url")
transfer = Transfer.api_create(context, DataSet.CATEGORIES[0],
access.value, test_org.guid,
file_source)
transfer.ensure_finished()
step("Get data set matching to transfer {}".format(transfer.title))
data_set = DataSet.api_get_matching_to_transfer(
org=test_org, transfer_title=transfer.title)
return transfer, data_set
def transfer_flow(transfer, core_org):
step("Check that the transfer is finished")
transfer.ensure_finished()
step("Check that the transfer is on the list")
transfers = Transfer.api_get_list(org_guid_list=[core_org.guid])
assert transfer in transfers
step("Get data set matching to transfer")
data_set = DataSet.api_get_matching_to_transfer(
org_guid=core_org.guid, transfer_title=transfer.title)
step("Delete the data set")
data_set.api_delete()
step("Check that the data set was deleted")
data_sets = DataSet.api_get_list(org_guid_list=[core_org.guid])
assert data_set not in data_sets
step("Delete the transfer")
transfer.api_delete()
step("Check that the transfer was deleted")
transfers = Transfer.api_get_list(org_guid_list=[core_org.guid])
assert transfer not in transfers
def test_visibility_of_events(self, test_org, context,
test_org_manager_client, test_data_urls):
events_before = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid)
step("Create dataset by admin")
transfer = Transfer.api_create(context,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url)
transfer.ensure_finished()
events_after = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid)
step("Check admin dataset creation event is visible")
assert len(events_before) + 1 == len(events_after)
step("Create dataset by non-admin user")
transfer = Transfer.api_create(context,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url,
client=test_org_manager_client)
transfer.ensure_finished()
events_after = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid, client=test_org_manager_client)
step("Check that non-admin dataset creation event is visible")
assert len(events_before) + 2 == len(events_after)
def test_transfer_and_dataset_are_not_visible_in_other_org(
self, context, core_org, test_org, test_data_urls):
step("Create transfer and get dataset")
transfer = self._create_transfer(context,
category=self.DEFAULT_CATEGORY,
org_guid=test_org.guid,
test_data_urls=test_data_urls)
dataset = DataSet.api_get_matching_to_transfer(
org_guid=test_org.guid, transfer_title=transfer.title)
step("Check transfer is not visible on other organization")
transfers = Transfer.api_get_list(org_guid_list=[core_org.guid])
assert transfer not in transfers
step("Check dataset is not visible on other organization")
datasets = DataSet.api_get_list(org_guid_list=[core_org.guid])
assert dataset not in datasets
def cleanup_test_data():
core_org_guid = core_org().guid
test_object_models = [
{'name': 'data set', 'objects_list': DataSet.api_get_list(), 'name_attribute': 'title'},
{'name': 'transfer', 'objects_list': Transfer.api_get_list(), 'name_attribute': 'title'},
{'name': 'user', 'objects_list': User.get_list_in_organization(org_guid=core_org_guid), 'name_attribute': 'username'},
{'name': 'invitation', 'objects_list': Invitation.api_get_list(), 'name_attribute': 'username'},
{'name': 'application', 'objects_list': Application.get_list(), 'name_attribute': 'name'},
{'name': 'service', 'objects_list': ServiceInstance.get_list(), 'name_attribute': 'name'},
{'name': 'offering', 'objects_list': ServiceOffering.get_list(), 'name_attribute': 'label'},
{'name': 'scoring engine model', 'objects_list': ScoringEngineModel.get_list(org_guid=core_org_guid),
'name_attribute': 'name'}
]
for model in test_object_models:
_cleanup_test_data(**model)
def test_10_latest_events_on_dashboard_the_same_as_in_LES(
self, context, test_org, test_data_urls):
step("Produce an event in the tested organization - create a data set")
transfer = Transfer.api_create(context,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url)
transfer.ensure_finished()
step("Retrieve latest events from dashboard")
dashboard_latest_events = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid)
step(
"Retrieve latest events from the LES, filtering with tested organization"
)
latest_events_response = LatestEvent.api_get_latest_events(
test_org.guid)
step("Check that dashboard contains 10 latest events from LES")
ten_latest_events = sorted(latest_events_response, reverse=True)[:10]
assert_unordered_list_equal(ten_latest_events, dashboard_latest_events)
def _create_transfer(self,
context,
org_guid,
column_count=10,
row_count=10,
category=TestSubmitTransfer.DEFAULT_CATEGORY,
size=None,
file_name=None):
step("Generate sample csv file")
file_path = generate_csv_file(column_count=column_count,
row_count=row_count,
size=size,
file_name=file_name)
step("Create a transfer with new category")
transfer = Transfer.api_create_by_file_upload(context,
category=category,
org_guid=org_guid,
file_path=file_path)
transfer.ensure_finished()
return transfer
def test_data_catalog_events(self, org, admin_user, events_before, context, test_data_urls):
step("Add admin to test org")
admin_user.api_add_to_organization(org_guid=org.guid)
step("Produce an event in the test org - create a dataset")
transfer = Transfer.api_create(context, org_guid=org.guid, source=test_data_urls.test_transfer.url)
transfer.ensure_finished()
step("Ensure that a dataset has been created")
data_set = DataSet.api_get_matching_to_transfer(transfer_title=transfer.title, org_guid=org.guid)
step("Retrieve latest events. Check that there is one new event related to dataset creation.")
events_after_create = LatestEvent.api_get_latest_events(org_guid=org.guid)
assert len(events_after_create) == len(events_before) + 1
self._assert_event_is_correct(events_after_create[0], category=EventCategory.data_catalog, org_guid=org.guid,
message=EventMessage.dataset_added.format(test_data_urls.test_transfer.url))
step("Produce an event in the test org - delete the dataset")
data_set.api_delete()
step("Retrieve latest events. Check that there is one new event related to dataset creation.")
events_after_delete = LatestEvent.api_get_latest_events(org_guid=org.guid)
assert len(events_after_delete) == len(events_after_create) + 1
self._assert_event_is_correct(events_after_delete[0], category=EventCategory.data_catalog, org_guid=org.guid,
message=EventMessage.dataset_deleted.format(test_data_urls.test_transfer.url))
def create_test_data_sets(cls, request, test_org, class_context,
test_data_urls):
step("Create new transfer for each category")
cls.transfers = []
for category in DataSet.CATEGORIES:
for dataset_privacy in DataSet.IS_PUBLIC:
cls.transfer = Transfer.api_create(
class_context,
category,
is_public=dataset_privacy,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url)
cls.transfers.append(cls.transfer)
cls.transfer.ensure_finished()
step("Get all data sets in the test org")
cls.transfer_titles = [t.title for t in cls.transfers]
dataset_list = DataSet.api_get_list(org_guid_list=[test_org.guid])
cls.datasets = [
d for d in dataset_list if d.title in cls.transfer_titles
]
def test_add_user(self, test_org, context):
""" Verify if user data are properly saved in HDFS while creating user and uploading data file to TAP.
After create user test verifies that user is synchronized. Then login to platform as new user.
Create transfer to HDFS and check that transfer is completed.
Input data:
user name: name for new user
data file: file which is transferred to HDFS
organization: organization wherein will be new user
Expected results:
It is possible to create new user and login to platform as new user. New user is able to create
transfer to HDFS. User data are properly saved on HDFS.
Steps:
1. Create new user
2. Check that user is created completely
3. Login as new registered user
4. Check that the user is able to create a transfer
5. Ensure that transfer is completed successfully
"""
step("Create new user")
user = User.create_by_adding_to_organization(
context=context,
org_guid=test_org.guid,
role=User.ORG_ROLE["user"])
step("Check that user was synchronized")
ag_user = AuthGatewayUser.get_user_state(org_guid=test_org.guid,
user_id=user.guid)
assert ag_user.is_synchronized, "User {} was not synchronized".format(
user.username)
step("Login new registered user")
client = user.login()
step("Check that the user is able to create a transfer")
transfer = Transfer.api_create_by_file_upload(
context,
test_org.guid,
file_utils.generate_csv_file(),
client=client)
transfer.ensure_finished()
def test_events_visibility_from_another_org(self, test_org, context,
another_org,
another_org_client):
events_before = LatestEvent.api_get_latest_events_from_org_metrics(
another_org.guid, client=another_org_client)
step("Create dataset in another org")
transfer = Transfer.api_create(context,
org_guid=another_org.guid,
source=Urls.test_transfer_link,
client=another_org_client)
transfer.ensure_finished()
step("Check event is on the latest events list")
events_after = LatestEvent.api_get_latest_events_from_org_metrics(
another_org.guid, client=another_org_client)
assert len(events_before) + 1 == len(
events_after), "The new event is not visible"
step("Check events from one org are not visible in another org")
test_org_events = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid)
assert all((event not in test_org_events for event in events_after)), \
"Some events from the another org are visible in first org"
def test_create_transfer_without_category(self, context, test_org):
"""
<b>Description:</b>
Check that transfer can be created from an url without providing a category.
<b>Input data:</b>
1. organization id
2. url with a source file
<b>Expected results:</b>
Transfer is successfully created and belongs to 'other' category.
<b>Steps:</b>
1. Create transfer without providing a category.
2. Check that newly created transfer has a 'other' category.
"""
transfer = self._create_transfer(context,
category=None,
org_guid=test_org.guid)
step("Get transfer and check it's category")
transfer_list = Transfer.api_get_list(org_guid_list=[test_org.guid])
transfer.category = "other"
assert transfer in transfer_list, "Transfer was not found"
logger.info("Removing databases:\n{}".format("\n".join(dbs)))
dbs = map(lambda name: "DROP DATABASE {} CASCADE;".format(name), dbs)
dbs = "".join(dbs)
hive.exec_query(dbs)
else:
logger.info("No database to remove.")
if __name__ == "__main__":
all_data_sets = DataSet.api_get_list()
test_data_sets = [x for x in all_data_sets if is_test_object_name(x.title)]
log_deleted_objects(test_data_sets, "data set")
fixtures.tear_down_test_objects(test_data_sets)
all_transfers = Transfer.api_get_list()
test_transfers = [x for x in all_transfers if is_test_object_name(x.title)]
log_deleted_objects(test_transfers, "transfer")
fixtures.tear_down_test_objects(test_transfers)
all_users = User.cf_api_get_all_users()
test_users = [x for x in all_users if is_test_object_name(x.username)]
log_deleted_objects(test_users, "user")
fixtures.tear_down_test_objects(test_users)
all_pending_invitations = Invitation.api_get_list()
test_invitations = [
x for x in all_pending_invitations if is_test_object_name(x.username)
]
log_deleted_objects(test_invitations, "invitation")
fixtures.tear_down_test_objects(test_invitations)
def test_admin_can_get_transfer_list(self, test_org):
step("Check if the list of transfers can be retrieved")
Transfer.api_get_list(org_guid_list=[test_org.guid])
