Example #1
0
    def setup_inputs_swift(self, clients, tenant_id, input_url, swift_files,
                           username, password):
        swift_scenario = swift_utils.SwiftScenario(clients=clients,
                                                   context=self.context)
        # TODO(astudenov): use self.generate_random_name()
        container_name = "rally_" + parse.urlparse(input_url).netloc.rstrip(
            ".sahara")
        self.context["sahara"]["container_name"] = (
            swift_scenario._create_container(container_name=container_name))
        for swift_file in swift_files:
            content = requests.get(swift_file["download_url"]).content
            self.context["sahara"]["swift_objects"].append(
                swift_scenario._upload_object(
                    self.context["sahara"]["container_name"],
                    content,
                    object_name=swift_file["name"]))
            input_ds_swift = clients.sahara().data_sources.create(
                name=self.generate_random_name(),
                description="",
                data_source_type="swift",
                url=input_url,
                credential_user=username,
                credential_pass=password)

            self.context["tenants"][tenant_id]["sahara"]["input"] = (
                input_ds_swift.id)
Example #2
0
 def consume(cache, args):
     container, tenant_containers = args
     user = container["user"]
     if user["id"] not in cache:
         cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
     cache[user["id"]]._delete_container(container["container"])
     tenant_containers.remove(container)
Example #3
0
    def test__upload_object(self):
        container_name = mock.MagicMock()
        object_name = mock.MagicMock()
        content = mock.MagicMock()
        etag = mock.MagicMock()
        self.clients("swift").put_object.return_value = etag
        scenario = utils.SwiftScenario()

        # container + content + name + kw
        self.assertEqual((etag, object_name),
                         scenario._upload_object(container_name,
                                                 content,
                                                 object_name=object_name,
                                                 fargs="f"))
        kw = {"fargs": "f"}
        self.clients("swift").put_object.assert_called_once_with(
            container_name, object_name, content, **kw)
        # container + content + kw
        scenario._generate_random_name = mock.MagicMock(
            return_value=object_name)
        self.clients("swift").put_object.reset_mock()
        self.assertEqual((etag, object_name),
                         scenario._upload_object(container_name,
                                                 content,
                                                 fargs="f"))
        kw = {"fargs": "f"}
        self.clients("swift").put_object.assert_called_once_with(
            container_name, object_name, content, **kw)
        self.assertEqual(1, scenario._generate_random_name.call_count)

        self._test_atomic_action_timer(scenario.atomic_actions(),
                                       "swift.upload_object")
Example #4
0
 def consume(cache, args):
     object_name, container = args
     user = container["user"]
     if user["id"] not in cache:
         cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
     cache[user["id"]]._delete_object(container["container"],
                                      object_name)
     container["objects"].remove(object_name)
Example #5
0
    def test__create_container(self):
        container_name = mock.MagicMock()
        scenario = utils.SwiftScenario()

        # name + public=True + kw
        self.assertEqual(
            container_name,
            scenario._create_container(container_name, public=True, fargs="f"))
        kw = {"headers": {"X-Container-Read": ".r:*,.rlistings"}, "fargs": "f"}
        self.clients("swift").put_container.assert_called_once_with(
            container_name, **kw)
        # name + public=True + additional header + kw
        self.clients("swift").put_container.reset_mock()
        self.assertEqual(
            container_name,
            scenario._create_container(container_name,
                                       public=True,
                                       headers={"X-fake-name": "fake-value"},
                                       fargs="f"))
        kw = {
            "headers": {
                "X-Container-Read": ".r:*,.rlistings",
                "X-fake-name": "fake-value"
            },
            "fargs": "f"
        }
        self.clients("swift").put_container.assert_called_once_with(
            container_name, **kw)
        # name + public=False + additional header + kw
        self.clients("swift").put_container.reset_mock()
        self.assertEqual(
            container_name,
            scenario._create_container(container_name,
                                       public=False,
                                       headers={"X-fake-name": "fake-value"},
                                       fargs="f"))
        kw = {"headers": {"X-fake-name": "fake-value"}, "fargs": "f"}
        self.clients("swift").put_container.assert_called_once_with(
            container_name, **kw)
        # name + kw
        self.clients("swift").put_container.reset_mock()
        self.assertEqual(container_name,
                         scenario._create_container(container_name, fargs="f"))
        kw = {"fargs": "f"}
        self.clients("swift").put_container.assert_called_once_with(
            container_name, **kw)
        # kw
        scenario._generate_random_name = mock.MagicMock(
            return_value=container_name)
        self.clients("swift").put_container.reset_mock()
        self.assertEqual(container_name, scenario._create_container(fargs="f"))
        kw = {"fargs": "f"}
        self.clients("swift").put_container.assert_called_once_with(
            container_name, **kw)
        self.assertEqual(1, scenario._generate_random_name.call_count)

        self._test_atomic_action_timer(scenario.atomic_actions(),
                                       "swift.create_container")
Example #6
0
 def consume(cache, args):
     user, tenant_containers = args
     if user["id"] not in cache:
         cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
     container_name = cache[user["id"]]._create_container()
     tenant_containers.append({"user": user,
                               "container": container_name,
                               "objects": []})
     containers.append((user["tenant_id"], container_name))
Example #7
0
    def test__delete_container(self):
        container_name = mock.MagicMock()
        scenario = utils.SwiftScenario()
        scenario._delete_container(container_name, fargs="f")

        kw = {"fargs": "f"}
        self.clients("swift").delete_container.assert_called_once_with(
            container_name, **kw)

        self._test_atomic_action_timer(scenario.atomic_actions(),
                                       "swift.delete_container")
Example #8
0
 def consume(cache, container):
     user = container["user"]
     if user["id"] not in cache:
         cache[user["id"]] = swift_utils.SwiftScenario(
             {"user": user})
     dummy_file.seek(0)
     object_name = cache[user["id"]]._upload_object(
         container["container"],
         dummy_file)[1]
     container["objects"].append(object_name)
     objects.append((user["tenant_id"], container["container"],
                     object_name))
Example #9
0
    def test__list_containers(self):
        headers_dict = mock.MagicMock()
        containers_list = mock.MagicMock()
        self.clients("swift").get_account.return_value = (headers_dict,
                                                          containers_list)
        scenario = utils.SwiftScenario(context=self.context)

        self.assertEqual((headers_dict, containers_list),
                         scenario._list_containers(fargs="f"))
        kw = {"full_listing": True, "fargs": "f"}
        self.clients("swift").get_account.assert_called_once_with(**kw)

        self._test_atomic_action_timer(scenario.atomic_actions(),
                                       "swift.list_containers")
Example #10
0
    def test__list_objects(self):
        container_name = mock.MagicMock()
        headers_dict = mock.MagicMock()
        objects_list = mock.MagicMock()
        self.clients("swift").get_container.return_value = (headers_dict,
                                                            objects_list)
        scenario = utils.SwiftScenario()

        self.assertEqual((headers_dict, objects_list),
                         scenario._list_objects(container_name, fargs="f"))
        kw = {"full_listing": True, "fargs": "f"}
        self.clients("swift").get_container.assert_called_once_with(
            container_name, **kw)

        self._test_atomic_action_timer(scenario.atomic_actions(),
                                       "swift.list_objects")
Example #11
0
    def test__download_object(self):
        container_name = mock.MagicMock()
        object_name = mock.MagicMock()
        headers_dict = mock.MagicMock()
        content = mock.MagicMock()
        self.clients("swift").get_object.return_value = (headers_dict, content)
        scenario = utils.SwiftScenario(context=self.context)

        self.assertEqual((headers_dict, content),
                         scenario._download_object(container_name, object_name,
                                                   fargs="f"))
        kw = {"fargs": "f"}
        self.clients("swift").get_object.assert_called_once_with(
            container_name, object_name,
            **kw)

        self._test_atomic_action_timer(scenario.atomic_actions(),
                                       "swift.download_object")
Example #12
0
    def test__create_container(self, public=True, kwargs=None, headers=None):
        if kwargs is None:
            kwargs = {"fakearg": "fake"}
        if headers is None:
            headers = {}
        scenario = utils.SwiftScenario(self.context)
        scenario.generate_random_name = mock.MagicMock()

        container = scenario._create_container(public=public,
                                               headers=headers,
                                               **kwargs)
        self.assertEqual(container,
                         scenario.generate_random_name.return_value)
        kwargs["headers"] = headers
        kwargs["headers"]["X-Container-Read"] = ".r:*,.rlistings"
        self.clients("swift").put_container.assert_called_once_with(
            scenario.generate_random_name.return_value,
            **kwargs)

        self._test_atomic_action_timer(scenario.atomic_actions(),
                                       "swift.create_container")
Example #13
0
    def setup(self):
        utils.init_sahara_context(self)
        for user, tenant_id in rutils.iterate_per_tenants(
                self.context["users"]):

            clients = osclients.Clients(user["credential"])
            sahara = clients.sahara()

            if self.config["output_type"] == "swift":
                swift = swift_utils.SwiftScenario(clients=clients,
                                                  context=self.context)
                container_name = self.generate_random_name()
                self.context["tenants"][tenant_id]["sahara"]["container"] = {
                    "name":
                    swift._create_container(container_name=container_name),
                    "output_swift_objects": []
                }
                self.setup_outputs_swift(swift, sahara, tenant_id,
                                         container_name,
                                         user["credential"].username,
                                         user["credential"].password)
            else:
                self.setup_outputs_hdfs(sahara, tenant_id,
                                        self.config["output_url_prefix"])