def test_submit_transfer_from_large_file(self, context, test_org):
"""
<b>Description:</b>
Check that transfer can be created from a csv file in any category.
<b>Input data:</b>
1. organization id
2. transfer category
3. test file
<b>Expected results:</b>
Test passes when transfer and dataset are successfully created from a csv file
and are visible in the test organization.
<b>Steps:</b>
1. Create transfer from a csv file.
2. Retrieve created transfer and dataset.
3. Check that created transfer and dataset are visible in the test organization.
"""
transfer = self._create_transfer(context,
org_guid=test_org.guid,
size=20 * 1024 * 1024)
step("Get data set matching to transfer {}".format(transfer.title))
DataSet.api_get_matching_to_transfer(org_guid=test_org.guid,
transfer_title=transfer.title)
def test_submit_transfer_from_large_file(self, context, test_org):
transfer = self._create_transfer(context,
org_guid=test_org.guid,
size=20 * 1024 * 1024)
step("Get data set matching to transfer {}".format(transfer.title))
DataSet.api_get_matching_to_transfer(org=test_org,
transfer_title=transfer.title)
def test_submit_transfer_from_empty_file(self, context, test_org):
transfer = self._create_transfer(context,
org_guid=test_org.guid,
category=self.DEFAULT_CATEGORY,
size=0)
step("Get data set matching to transfer {}".format(transfer.title))
DataSet.api_get_matching_to_transfer(org=test_org,
transfer_title=transfer.title)
def test_submit_transfer_from_file_with_space_in_name(
self, context, test_org):
transfer = self._create_transfer(
context,
org_guid=test_org.guid,
file_name="file with space in name {}.csv")
step("Get data set matching to transfer {}".format(transfer.title))
DataSet.api_get_matching_to_transfer(org=test_org,
transfer_title=transfer.title)
def test_3_create_data_set_and_publish_it_in_hive(self, class_context, test_org):
step("Create transfer and check it's finished")
transfer = Transfer.api_create(class_context, source=Urls.test_transfer_link, org_guid=test_org.guid,
title=self.transfer_title)
transfer.ensure_finished()
step("Publish in hive the data set created based on the submitted transfer")
data_set = DataSet.api_get_matching_to_transfer(org=test_org, transfer_title=self.transfer_title)
data_set.api_publish()
self.__class__.data_set_hdfs_path = ATKtools.dataset_uri_to_atk_uri(data_set.target_uri)
def dataset_target_uri(self, test_org, test_space, class_context, admin_user):
step("Add admin to space with developer role")
admin_user.api_add_to_space(test_space.guid, test_org.guid, User.SPACE_ROLES["developer"])
step("Create transfer")
transfer = Transfer.api_create(class_context, org_guid=test_org.guid, source=Urls.test_transfer_link)
transfer.ensure_finished()
step("Get dataset")
dataset = DataSet.api_get_matching_to_transfer(transfer.title, test_org)
return dataset.target_uri
def _get_transfer_and_dataset(self, test_org, file_source, access,
context):
step("Create transfer by providing a csv from url")
transfer = Transfer.api_create(context, DataSet.CATEGORIES[0],
access.value, test_org.guid,
file_source)
transfer.ensure_finished()
step("Get data set matching to transfer {}".format(transfer.title))
data_set = DataSet.api_get_matching_to_transfer(
org=test_org, transfer_title=transfer.title)
return transfer, data_set
def test_submit_transfer_from_empty_file(self, context, test_org):
"""
<b>Description:</b>
Check that it's possible to create transfer from an empty file.
<b>Input data:</b>
1. organization id
2. empty file
<b>Expected results:</b>
Test passes when transfer is created.
<b>Steps:</b>
1. Create transfer.
2. Get matching dataset of created transfer.
"""
transfer = self._create_transfer(context,
org_guid=test_org.guid,
category=self.DEFAULT_CATEGORY,
size=0)
step("Get data set matching to transfer {}".format(transfer.title))
DataSet.api_get_matching_to_transfer(org_guid=test_org.guid,
transfer_title=transfer.title)
def test_transfer_and_dataset_are_not_visible_in_other_org(
self, context, core_org, test_org, test_data_urls):
step("Create transfer and get dataset")
transfer = self._create_transfer(context,
category=self.DEFAULT_CATEGORY,
org_guid=test_org.guid,
test_data_urls=test_data_urls)
dataset = DataSet.api_get_matching_to_transfer(
org_guid=test_org.guid, transfer_title=transfer.title)
step("Check transfer is not visible on other organization")
transfers = Transfer.api_get_list(org_guid_list=[core_org.guid])
assert transfer not in transfers
step("Check dataset is not visible on other organization")
datasets = DataSet.api_get_list(org_guid_list=[core_org.guid])
assert dataset not in datasets
def test_submit_transfer_from_file_with_space_in_name(
self, context, test_org):
"""
<b>Description:</b>
Check that it's possible to create transfer from a file that contains spaces in name.
<b>Input data:</b>
1. organization id
2. file name with spaces
<b>Expected results:</b>
Test passes when transfer is created.
<b>Steps:</b>
1. Create transfer.
2. Get matching dataset of created transfer
"""
transfer = self._create_transfer(
context,
org_guid=test_org.guid,
file_name="file with space in name {}.csv")
step("Get data set matching to transfer {}".format(transfer.title))
DataSet.api_get_matching_to_transfer(org_guid=test_org.guid,
transfer_title=transfer.title)
def test_submit_and_delete_transfer(self, context, test_org):
"""
<b>Description:</b>
Check transfer creation from an url and deletion. Also check dataset deletion.
<b>Input data:</b>
1. organization guid
2. transfer category
3. url with a source file
<b>Expected results:</b>
Transfer is successfully created and deleted. Dataset is successfully deleted.
<b>Steps:</b>
1. Create transfer.
2. Retrieve corresponding dataset,
3. Delete transfer.
4. Check that transfer disappeared from the transfer list.
5. Delete dataset.
6. Check that dataset disappeared from the dataset list.
"""
transfer = self._create_transfer(context,
category=self.DEFAULT_CATEGORY,
org_guid=test_org.guid)
step(
"Get transfers and check if they are the same as the uploaded ones"
)
retrieved_transfer = Transfer.api_get(transfer.id)
assert transfer == retrieved_transfer, "The transfer is not the same"
dataset = DataSet.api_get_matching_to_transfer(
org_guid=test_org.guid, transfer_title=transfer.title)
TestSubmitTransfer.check_transfer_and_dataset_are_visible_in_test_org(
transfer=transfer, dataset=dataset, test_org=test_org)
step("Delete transfer")
transfer.cleanup()
step("Check transfer is not visible on list of transfers")
transfers = Transfer.api_get_list(org_guid_list=[test_org.guid])
assert transfer not in transfers
step("Delete dataset")
dataset.cleanup()
step("Check dataset is not visible on list of datasets")
datasets = DataSet.api_get_list(org_guid_list=[test_org.guid])
assert dataset not in datasets
def test_data_catalog_events(self, org, admin_user, events_before, context, test_data_urls):
step("Add admin to test org")
admin_user.api_add_to_organization(org_guid=org.guid)
step("Produce an event in the test org - create a dataset")
transfer = Transfer.api_create(context, org_guid=org.guid, source=test_data_urls.test_transfer.url)
transfer.ensure_finished()
step("Ensure that a dataset has been created")
data_set = DataSet.api_get_matching_to_transfer(transfer_title=transfer.title, org_guid=org.guid)
step("Retrieve latest events. Check that there is one new event related to dataset creation.")
events_after_create = LatestEvent.api_get_latest_events(org_guid=org.guid)
assert len(events_after_create) == len(events_before) + 1
self._assert_event_is_correct(events_after_create[0], category=EventCategory.data_catalog, org_guid=org.guid,
message=EventMessage.dataset_added.format(test_data_urls.test_transfer.url))
step("Produce an event in the test org - delete the dataset")
data_set.api_delete()
step("Retrieve latest events. Check that there is one new event related to dataset creation.")
events_after_delete = LatestEvent.api_get_latest_events(org_guid=org.guid)
assert len(events_after_delete) == len(events_after_create) + 1
self._assert_event_is_correct(events_after_delete[0], category=EventCategory.data_catalog, org_guid=org.guid,
message=EventMessage.dataset_deleted.format(test_data_urls.test_transfer.url))
def transfer_flow(transfer, core_org):
step("Check that the transfer is finished")
transfer.ensure_finished()
step("Check that the transfer is on the list")
transfers = Transfer.api_get_list(org_guid_list=[core_org.guid])
assert transfer in transfers
step("Get data set matching to transfer")
data_set = DataSet.api_get_matching_to_transfer(
org_guid=core_org.guid, transfer_title=transfer.title)
step("Delete the data set")
data_set.api_delete()
step("Check that the data set was deleted")
data_sets = DataSet.api_get_list(org_guid_list=[core_org.guid])
assert data_set not in data_sets
step("Delete the transfer")
transfer.api_delete()
step("Check that the transfer was deleted")
transfers = Transfer.api_get_list(org_guid_list=[core_org.guid])
assert transfer not in transfers