def create_test_data_sets(cls, request, test_org, class_context,
test_data_urls):
step("Create new transfer for each category")
cls.transfers = []
for category in DataSet.CATEGORIES:
cls.transfers.append(
Transfer.api_create(class_context,
category,
is_public=False,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url))
for category in DataSet.CATEGORIES:
cls.transfers.append(
Transfer.api_create(class_context,
category,
is_public=True,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url))
step("Ensure that transfers are finished")
for transfer in cls.transfers:
transfer.ensure_finished()
step("Get all data sets in the test org")
cls.transfer_titles = [t.title for t in cls.transfers]
dataset_list = DataSet.api_get_list(org_guid_list=[test_org.guid])
cls.datasets = [
d for d in dataset_list if d.title in cls.transfer_titles
]
def test_add_and_delete_transfer_from_link(core_org, context, test_data_urls):
"""
<b>Description:</b>
Checks if a file can be downloaded from a link to the platform.
<b>Input data:</b>
1. File URL.
2. Transfer title.
3. Transfer category.
<b>Expected results:</b>
Test passes when a file from link was downloaded to the platform and corresponding transfer and dataset were
successfully created and removed from the platform.
<b>Steps:</b>
1. Create a transfer using csv file.
2. Check if transfer finished with correct state and verify that it is present on the transfers list.
3. Check if the transfer has corresponding dataset.
4. Delete dataset and transfer from the platform.
5. Verify that transfer and dataset were removed from the platform.
"""
step("Create a transfer")
transfer = Transfer.api_create(context,
category="other",
source=test_data_urls.test_transfer.url,
org_guid=core_org.guid)
transfer_flow(transfer, core_org)
def _create_transfer(self, context, org_guid, category):
step("Create new transfer and wait until it's finished")
transfer = Transfer.api_create(context,
category=category,
source=Urls.test_transfer_link,
org_guid=org_guid)
transfer.ensure_finished()
return transfer
def test_add_and_delete_transfer_from_link(core_org, context):
"""Create and Delete Transfer from Link"""
step("Create a transfer")
transfer = Transfer.api_create(context,
category="other",
source=Urls.test_transfer_link,
org_guid=core_org.guid)
transfer_flow(transfer, core_org)
def test_3_create_data_set_and_publish_it_in_hive(self, class_context, test_org):
step("Create transfer and check it's finished")
transfer = Transfer.api_create(class_context, source=Urls.test_transfer_link, org_guid=test_org.guid,
title=self.transfer_title)
transfer.ensure_finished()
step("Publish in hive the data set created based on the submitted transfer")
data_set = DataSet.api_get_matching_to_transfer(org=test_org, transfer_title=self.transfer_title)
data_set.api_publish()
self.__class__.data_set_hdfs_path = ATKtools.dataset_uri_to_atk_uri(data_set.target_uri)
def dataset_target_uri(self, test_org, test_space, class_context, admin_user):
step("Add admin to space with developer role")
admin_user.api_add_to_space(test_space.guid, test_org.guid, User.SPACE_ROLES["developer"])
step("Create transfer")
transfer = Transfer.api_create(class_context, org_guid=test_org.guid, source=Urls.test_transfer_link)
transfer.ensure_finished()
step("Get dataset")
dataset = DataSet.api_get_matching_to_transfer(transfer.title, test_org)
return dataset.target_uri
def _get_transfer_and_dataset(self, test_org, file_source, access,
context):
step("Create transfer by providing a csv from url")
transfer = Transfer.api_create(context, DataSet.CATEGORIES[0],
access.value, test_org.guid,
file_source)
transfer.ensure_finished()
step("Get data set matching to transfer {}".format(transfer.title))
data_set = DataSet.api_get_matching_to_transfer(
org=test_org, transfer_title=transfer.title)
return transfer, data_set
def test_visibility_of_events(self, test_org, context,
test_org_manager_client, test_data_urls):
events_before = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid)
step("Create dataset by admin")
transfer = Transfer.api_create(context,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url)
transfer.ensure_finished()
events_after = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid)
step("Check admin dataset creation event is visible")
assert len(events_before) + 1 == len(events_after)
step("Create dataset by non-admin user")
transfer = Transfer.api_create(context,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url,
client=test_org_manager_client)
transfer.ensure_finished()
events_after = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid, client=test_org_manager_client)
step("Check that non-admin dataset creation event is visible")
assert len(events_before) + 2 == len(events_after)
def test_10_latest_events_on_dashboard_the_same_as_in_LES(
self, context, test_org, test_data_urls):
step("Produce an event in the tested organization - create a data set")
transfer = Transfer.api_create(context,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url)
transfer.ensure_finished()
step("Retrieve latest events from dashboard")
dashboard_latest_events = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid)
step(
"Retrieve latest events from the LES, filtering with tested organization"
)
latest_events_response = LatestEvent.api_get_latest_events(
test_org.guid)
step("Check that dashboard contains 10 latest events from LES")
ten_latest_events = sorted(latest_events_response, reverse=True)[:10]
assert_unordered_list_equal(ten_latest_events, dashboard_latest_events)
def test_data_catalog_events(self, org, admin_user, events_before, context, test_data_urls):
step("Add admin to test org")
admin_user.api_add_to_organization(org_guid=org.guid)
step("Produce an event in the test org - create a dataset")
transfer = Transfer.api_create(context, org_guid=org.guid, source=test_data_urls.test_transfer.url)
transfer.ensure_finished()
step("Ensure that a dataset has been created")
data_set = DataSet.api_get_matching_to_transfer(transfer_title=transfer.title, org_guid=org.guid)
step("Retrieve latest events. Check that there is one new event related to dataset creation.")
events_after_create = LatestEvent.api_get_latest_events(org_guid=org.guid)
assert len(events_after_create) == len(events_before) + 1
self._assert_event_is_correct(events_after_create[0], category=EventCategory.data_catalog, org_guid=org.guid,
message=EventMessage.dataset_added.format(test_data_urls.test_transfer.url))
step("Produce an event in the test org - delete the dataset")
data_set.api_delete()
step("Retrieve latest events. Check that there is one new event related to dataset creation.")
events_after_delete = LatestEvent.api_get_latest_events(org_guid=org.guid)
assert len(events_after_delete) == len(events_after_create) + 1
self._assert_event_is_correct(events_after_delete[0], category=EventCategory.data_catalog, org_guid=org.guid,
message=EventMessage.dataset_deleted.format(test_data_urls.test_transfer.url))
def create_test_data_sets(cls, request, test_org, class_context,
test_data_urls):
step("Create new transfer for each category")
cls.transfers = []
for category in DataSet.CATEGORIES:
for dataset_privacy in DataSet.IS_PUBLIC:
cls.transfer = Transfer.api_create(
class_context,
category,
is_public=dataset_privacy,
org_guid=test_org.guid,
source=test_data_urls.test_transfer.url)
cls.transfers.append(cls.transfer)
cls.transfer.ensure_finished()
step("Get all data sets in the test org")
cls.transfer_titles = [t.title for t in cls.transfers]
dataset_list = DataSet.api_get_list(org_guid_list=[test_org.guid])
cls.datasets = [
d for d in dataset_list if d.title in cls.transfer_titles
]
def test_events_visibility_from_another_org(self, test_org, context,
another_org,
another_org_client):
events_before = LatestEvent.api_get_latest_events_from_org_metrics(
another_org.guid, client=another_org_client)
step("Create dataset in another org")
transfer = Transfer.api_create(context,
org_guid=another_org.guid,
source=Urls.test_transfer_link,
client=another_org_client)
transfer.ensure_finished()
step("Check event is on the latest events list")
events_after = LatestEvent.api_get_latest_events_from_org_metrics(
another_org.guid, client=another_org_client)
assert len(events_before) + 1 == len(
events_after), "The new event is not visible"
step("Check events from one org are not visible in another org")
test_org_events = LatestEvent.api_get_latest_events_from_org_metrics(
test_org.guid)
assert all((event not in test_org_events for event in events_after)), \
"Some events from the another org are visible in first org"