def setup_inputs_swift(self, clients, tenant_id, input_url, swift_files, username, password): swift_scenario = swift_utils.SwiftScenario(clients=clients, context=self.context) # TODO(astudenov): use self.generate_random_name() container_name = "rally_" + parse.urlparse(input_url).netloc.rstrip( ".sahara") self.context["sahara"]["container_name"] = ( swift_scenario._create_container(container_name=container_name)) for swift_file in swift_files: content = requests.get(swift_file["download_url"]).content self.context["sahara"]["swift_objects"].append( swift_scenario._upload_object( self.context["sahara"]["container_name"], content, object_name=swift_file["name"])) input_ds_swift = clients.sahara().data_sources.create( name=self.generate_random_name(), description="", data_source_type="swift", url=input_url, credential_user=username, credential_pass=password) self.context["tenants"][tenant_id]["sahara"]["input"] = ( input_ds_swift.id)
def test__delete_container(self): container_name = mock.MagicMock() scenario = utils.SwiftScenario(context=self.context) scenario._delete_container(container_name, fargs="f") kw = {"fargs": "f"} self.clients("swift").delete_container.assert_called_once_with( container_name, **kw) self._test_atomic_action_timer(scenario.atomic_actions(), "swift.delete_container")
def consume(cache, args): container, tenant_containers = args user = container["user"] if user["id"] not in cache: cache[user["id"]] = swift_utils.SwiftScenario({ "user": user, "task": context.get("task", {}) }) cache[user["id"]]._delete_container(container["container"]) tenant_containers.remove(container)
def consume(cache, args): object_name, container = args user = container["user"] if user["id"] not in cache: cache[user["id"]] = swift_utils.SwiftScenario({ "user": user, "task": context.get("task", {}) }) cache[user["id"]]._delete_object(container["container"], object_name) container["objects"].remove(object_name)
def test__list_containers(self): headers_dict = mock.MagicMock() containers_list = mock.MagicMock() self.clients("swift").get_account.return_value = (headers_dict, containers_list) scenario = utils.SwiftScenario(context=self.context) self.assertEqual((headers_dict, containers_list), scenario._list_containers(fargs="f")) kw = {"full_listing": True, "fargs": "f"} self.clients("swift").get_account.assert_called_once_with(**kw) self._test_atomic_action_timer(scenario.atomic_actions(), "swift.list_containers")
def consume(cache, container): user = container["user"] if user["id"] not in cache: cache[user["id"]] = swift_utils.SwiftScenario({ "user": user, "task": context.get("task", {}) }) dummy_file.seek(0) object_name = cache[user["id"]]._upload_object( container["container"], dummy_file)[1] container["objects"].append(object_name) objects.append( (user["tenant_id"], container["container"], object_name))
def consume(cache, args): user, tenant_containers = args if user["id"] not in cache: cache[user["id"]] = swift_utils.SwiftScenario({ "user": user, "task": context.get("task", {}) }) container_name = cache[user["id"]]._create_container() tenant_containers.append({ "user": user, "container": container_name, "objects": [] }) containers.append((user["tenant_id"], container_name))
def test__download_object(self): container_name = mock.MagicMock() object_name = mock.MagicMock() headers_dict = mock.MagicMock() content = mock.MagicMock() self.clients("swift").get_object.return_value = (headers_dict, content) scenario = utils.SwiftScenario(context=self.context) self.assertEqual((headers_dict, content), scenario._download_object(container_name, object_name, fargs="f")) kw = {"fargs": "f"} self.clients("swift").get_object.assert_called_once_with( container_name, object_name, **kw) self._test_atomic_action_timer(scenario.atomic_actions(), "swift.download_object")
def test__create_container(self, public=True, kwargs=None, headers=None): if kwargs is None: kwargs = {"fakearg": "fake"} if headers is None: headers = {} scenario = utils.SwiftScenario(self.context) scenario.generate_random_name = mock.MagicMock() container = scenario._create_container(public=public, headers=headers, **kwargs) self.assertEqual(container, scenario.generate_random_name.return_value) kwargs["headers"] = headers kwargs["headers"]["X-Container-Read"] = ".r:*,.rlistings" self.clients("swift").put_container.assert_called_once_with( scenario.generate_random_name.return_value, **kwargs) self._test_atomic_action_timer(scenario.atomic_actions(), "swift.create_container")
def test__upload_object(self): container_name = mock.MagicMock() content = mock.MagicMock() etag = mock.MagicMock() self.clients("swift").put_object.return_value = etag scenario = utils.SwiftScenario(self.context) scenario.generate_random_name = mock.MagicMock() self.clients("swift").put_object.reset_mock() self.assertEqual((etag, scenario.generate_random_name.return_value), scenario._upload_object(container_name, content, fargs="f")) kw = {"fargs": "f"} self.clients("swift").put_object.assert_called_once_with( container_name, scenario.generate_random_name.return_value, content, **kw) self.assertEqual(1, scenario.generate_random_name.call_count) self._test_atomic_action_timer(scenario.atomic_actions(), "swift.upload_object")
def setup(self): utils.init_sahara_context(self) for user, tenant_id in rutils.iterate_per_tenants( self.context["users"]): clients = osclients.Clients(user["credential"]) sahara = clients.sahara() if self.config["output_type"] == "swift": swift = swift_utils.SwiftScenario(clients=clients, context=self.context) container_name = self.generate_random_name() self.context["tenants"][tenant_id]["sahara"]["container"] = { "name": swift._create_container(container_name=container_name), "output_swift_objects": [] } self.setup_outputs_swift(swift, sahara, tenant_id, container_name, user["credential"].username, user["credential"].password) else: self.setup_outputs_hdfs(sahara, tenant_id, self.config["output_url_prefix"])