def test_create_dataset_from_hdfs_uri(self, context, test_org, test_data_urls): """ <b>Description:</b> Check that dataset can be created by passing hdfs uri as source. <b>Input data:</b> 1. organization id 2. hdfs URI <b>Expected results:</b> Test passes when dataset is successfully created and it has a hdfs uri in source_uri property. <b>Steps:</b> 1. Create dataset from an url. 3. Get dataset target_uri (hdfs path) and create new dataset with it. 4. Compare second dataset source_uri is the same as first dataset target_uri. """ step("Create source dataset") _, source_dataset = data_catalog.create_dataset_from_link( context, test_org.guid, test_data_urls.test_transfer.url) step("Create dataset from hdfs uri") _, dataset = data_catalog.create_dataset_from_link( context, test_org.guid, source_dataset.target_uri) assert dataset.source_uri == source_dataset.target_uri
def test_create_dataset_from_hdfs_uri(self, context, test_org): step("Create source dataset") _, source_dataset = data_catalog.create_dataset_from_link( context, test_org, Urls.test_transfer_link) step("Create dataset from hdfs uri") _, dataset = data_catalog.create_dataset_from_link( context, test_org, source_dataset.target_uri) assert dataset.source_uri == source_dataset.target_uri
def test_1_create_transfer_and_dataset_from_hadoop_mapreduce_examples( self, class_context, test_org, add_admin_to_test_org, test_data_urls): """ <b>Description:</b> Create transfer and dataset from hadoop mapreduce examples. <b>Input data:</b> 1. organization id 2. Url with hadoop mapreduce examples <b>Expected results:</b> Test passes when transfer and dataset are created successfully with hadoop mapreduce examples and hdfs path is retrieved. <b>Steps:</b> 1. Create transfer and dataset from hadoop mapreduce examples. 2. Retrieve a hdfs source file path """ step("Create dataset from hadoop mapreduce examples jar") _, dataset = create_dataset_from_link( context=class_context, org_guid=test_org.guid, source=test_data_urls.hadoop_mapreduce_examples.url) step("Create jar path") self.__class__.jar_path = dataset.target_uri.replace( "hdfs://nameservice1", "")
def dataset(cls, request, test_org, add_admin_to_test_org, class_context, test_data_urls): _, dataset = data_catalog.create_dataset_from_link( class_context, org_guid=test_org.guid, source=test_data_urls.test_transfer.url) return dataset
def test_create_transfer_from_atk_model_file(self, context, test_org, ref_space, atk_virtualenv, initial_dataset): step("Get atk app from core space") atk_app = next( (app for app in Application.cf_api_get_list_by_space(ref_space.guid) if app.name == "atk"), None) if atk_app is None: raise AssertionError("Atk app not found in core space") step("Install atk client package in virtualenv") atk_virtualenv.create() atk_virtualenv.pip_install(ATKtools.get_atk_client_url( atk_app.urls[0])) step("Run atk create model script") atk_test_script_path = os.path.join("fixtures", "atk_test_scripts", "atk_create_model.py") response = atk_virtualenv.run_atk_script( atk_test_script_path, atk_app.urls[0], arguments={"--target_uri": initial_dataset.target_uri}) step("Retrieve path to model file created by atk") hdfs_model_path = response.split("hdfs_model_path: ", 1)[1] step("Create dataset by providing retrieved model file path") _, ds = data_catalog.create_dataset_from_link(context, org=test_org, source=hdfs_model_path) assert ds.source_uri == hdfs_model_path
def dataset(self, test_org, context, test_data_urls): step("Create data set") _, dataset = data_catalog.create_dataset_from_link( context, org_guid=test_org.guid, source=test_data_urls.test_transfer.url) return dataset
def test_1_check_dataset_from_HDFS(self, test_org): datasets = data_catalog.create_datasets_from_links(self.context, test_org.guid, [self.HDFS_CONFIG_DIR + source for source in self.HDFS_CONFIG_FILES]) assertions.assert_datasets_not_empty(datasets) _, self.__class__.TEST_DATASET = data_catalog.create_dataset_from_link( self.context, test_org.guid, self.HDFS_OUTPUT_DIR + self.HDFS_OUTPUT_FILES[0]) assert self.TEST_DATASET.size > 0
def model_hdfs_path(request, test_org, add_admin_to_test_org): log_fixture("Create a transfer and get hdfs path") context = Context() _, data_set = data_catalog.create_dataset_from_link(context, org=test_org, source=Urls.model_url) request.addfinalizer(lambda: context.cleanup()) return data_set.target_uri
def test_2_create_dataset(self, class_context, core_hdfs_instance): step("Create dataset from url") _, self.__class__.dataset = create_dataset_from_link( class_context, self.test_org, Urls.test_transfer_link, client=self.test_org_manager_client) step("Check dataset directory in hdfs") hdfs_dirs = self._list_hdfs_directories( self.USERSPACE_PATH.format(self.test_org.guid, core_hdfs_instance.guid)) self._assert_directory_in_hdfs(hdfs_dirs, name=self.dataset.object_store_id, owner=self.test_org_manager.guid, group=self.test_org.guid)
def test_0_create_transfer_and_dataset_with_csv_link( self, class_context, test_org, test_data_urls): """ <b>Description:</b> Create transfer and dataset from an url and publish created dataset in hue. <b>Input data:</b> 1. test organization <b>Expected results:</b> Test passes when transfer and dataset are successfully created and dataset is published in hue. <b>Steps:</b> 1. Create transfer and dataset. 2. Publish created dataset in hue. """ step("Create new transfer and dataset") self.__class__.transfer, dataset = create_dataset_from_link( context=class_context, org_guid=test_org.guid, source=test_data_urls.test_transfer.url) step("Publish dataset in HUE") dataset.api_publish()
def dataset(self, test_org, add_admin_to_test_org, context): step("Create data set") _, dataset = data_catalog.create_dataset_from_link( context, org=test_org, source=Urls.test_transfer_link) return dataset
def assert_dataset_greater_with_retry(value_to_compare, *args, **kwargs): _, dataset = data_catalog.create_dataset_from_link(*args, **kwargs) assert dataset.size > value_to_compare
def dataset_target_uri(self, test_org, class_context, add_admin_to_test_org, test_data_urls): _, dataset = create_dataset_from_link(class_context, test_org.guid, test_data_urls.test_transfer.url) return dataset.target_uri
def dataset(cls, request, test_org, add_admin_to_test_org, class_context): _, dataset = data_catalog.create_dataset_from_link( class_context, org=test_org, source=Urls.test_transfer_link) return dataset