def setup_inputs_swift(self, clients, tenant_id, input_url, swift_files,
username, password):
swift_scenario = swift_utils.SwiftScenario(clients=clients,
context=self.context)
# TODO(astudenov): use self.generate_random_name()
container_name = "rally_" + parse.urlparse(input_url).netloc.rstrip(
".sahara")
self.context["sahara"]["container_name"] = (
swift_scenario._create_container(container_name=container_name))
for swift_file in swift_files:
content = requests.get(swift_file["download_url"]).content
self.context["sahara"]["swift_objects"].append(
swift_scenario._upload_object(
self.context["sahara"]["container_name"],
content,
object_name=swift_file["name"]))
input_ds_swift = clients.sahara().data_sources.create(
name=self.generate_random_name(),
description="",
data_source_type="swift",
url=input_url,
credential_user=username,
credential_pass=password)
self.context["tenants"][tenant_id]["sahara"]["input"] = (
input_ds_swift.id)
def consume(cache, args):
container, tenant_containers = args
user = container["user"]
if user["id"] not in cache:
cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
cache[user["id"]]._delete_container(container["container"])
tenant_containers.remove(container)
def test__upload_object(self):
container_name = mock.MagicMock()
object_name = mock.MagicMock()
content = mock.MagicMock()
etag = mock.MagicMock()
self.clients("swift").put_object.return_value = etag
scenario = utils.SwiftScenario()
# container + content + name + kw
self.assertEqual((etag, object_name),
scenario._upload_object(container_name,
content,
object_name=object_name,
fargs="f"))
kw = {"fargs": "f"}
self.clients("swift").put_object.assert_called_once_with(
container_name, object_name, content, **kw)
# container + content + kw
scenario._generate_random_name = mock.MagicMock(
return_value=object_name)
self.clients("swift").put_object.reset_mock()
self.assertEqual((etag, object_name),
scenario._upload_object(container_name,
content,
fargs="f"))
kw = {"fargs": "f"}
self.clients("swift").put_object.assert_called_once_with(
container_name, object_name, content, **kw)
self.assertEqual(1, scenario._generate_random_name.call_count)
self._test_atomic_action_timer(scenario.atomic_actions(),
"swift.upload_object")
def consume(cache, args):
object_name, container = args
user = container["user"]
if user["id"] not in cache:
cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
cache[user["id"]]._delete_object(container["container"],
object_name)
container["objects"].remove(object_name)
def test__create_container(self):
container_name = mock.MagicMock()
scenario = utils.SwiftScenario()
# name + public=True + kw
self.assertEqual(
container_name,
scenario._create_container(container_name, public=True, fargs="f"))
kw = {"headers": {"X-Container-Read": ".r:*,.rlistings"}, "fargs": "f"}
self.clients("swift").put_container.assert_called_once_with(
container_name, **kw)
# name + public=True + additional header + kw
self.clients("swift").put_container.reset_mock()
self.assertEqual(
container_name,
scenario._create_container(container_name,
public=True,
headers={"X-fake-name": "fake-value"},
fargs="f"))
kw = {
"headers": {
"X-Container-Read": ".r:*,.rlistings",
"X-fake-name": "fake-value"
},
"fargs": "f"
}
self.clients("swift").put_container.assert_called_once_with(
container_name, **kw)
# name + public=False + additional header + kw
self.clients("swift").put_container.reset_mock()
self.assertEqual(
container_name,
scenario._create_container(container_name,
public=False,
headers={"X-fake-name": "fake-value"},
fargs="f"))
kw = {"headers": {"X-fake-name": "fake-value"}, "fargs": "f"}
self.clients("swift").put_container.assert_called_once_with(
container_name, **kw)
# name + kw
self.clients("swift").put_container.reset_mock()
self.assertEqual(container_name,
scenario._create_container(container_name, fargs="f"))
kw = {"fargs": "f"}
self.clients("swift").put_container.assert_called_once_with(
container_name, **kw)
# kw
scenario._generate_random_name = mock.MagicMock(
return_value=container_name)
self.clients("swift").put_container.reset_mock()
self.assertEqual(container_name, scenario._create_container(fargs="f"))
kw = {"fargs": "f"}
self.clients("swift").put_container.assert_called_once_with(
container_name, **kw)
self.assertEqual(1, scenario._generate_random_name.call_count)
self._test_atomic_action_timer(scenario.atomic_actions(),
"swift.create_container")
def consume(cache, args):
user, tenant_containers = args
if user["id"] not in cache:
cache[user["id"]] = swift_utils.SwiftScenario({"user": user})
container_name = cache[user["id"]]._create_container()
tenant_containers.append({"user": user,
"container": container_name,
"objects": []})
containers.append((user["tenant_id"], container_name))
def test__delete_container(self):
container_name = mock.MagicMock()
scenario = utils.SwiftScenario()
scenario._delete_container(container_name, fargs="f")
kw = {"fargs": "f"}
self.clients("swift").delete_container.assert_called_once_with(
container_name, **kw)
self._test_atomic_action_timer(scenario.atomic_actions(),
"swift.delete_container")
def consume(cache, container):
user = container["user"]
if user["id"] not in cache:
cache[user["id"]] = swift_utils.SwiftScenario(
{"user": user})
dummy_file.seek(0)
object_name = cache[user["id"]]._upload_object(
container["container"],
dummy_file)[1]
container["objects"].append(object_name)
objects.append((user["tenant_id"], container["container"],
object_name))
def test__list_containers(self):
headers_dict = mock.MagicMock()
containers_list = mock.MagicMock()
self.clients("swift").get_account.return_value = (headers_dict,
containers_list)
scenario = utils.SwiftScenario(context=self.context)
self.assertEqual((headers_dict, containers_list),
scenario._list_containers(fargs="f"))
kw = {"full_listing": True, "fargs": "f"}
self.clients("swift").get_account.assert_called_once_with(**kw)
self._test_atomic_action_timer(scenario.atomic_actions(),
"swift.list_containers")
def test__list_objects(self):
container_name = mock.MagicMock()
headers_dict = mock.MagicMock()
objects_list = mock.MagicMock()
self.clients("swift").get_container.return_value = (headers_dict,
objects_list)
scenario = utils.SwiftScenario()
self.assertEqual((headers_dict, objects_list),
scenario._list_objects(container_name, fargs="f"))
kw = {"full_listing": True, "fargs": "f"}
self.clients("swift").get_container.assert_called_once_with(
container_name, **kw)
self._test_atomic_action_timer(scenario.atomic_actions(),
"swift.list_objects")
def test__download_object(self):
container_name = mock.MagicMock()
object_name = mock.MagicMock()
headers_dict = mock.MagicMock()
content = mock.MagicMock()
self.clients("swift").get_object.return_value = (headers_dict, content)
scenario = utils.SwiftScenario(context=self.context)
self.assertEqual((headers_dict, content),
scenario._download_object(container_name, object_name,
fargs="f"))
kw = {"fargs": "f"}
self.clients("swift").get_object.assert_called_once_with(
container_name, object_name,
**kw)
self._test_atomic_action_timer(scenario.atomic_actions(),
"swift.download_object")
def test__create_container(self, public=True, kwargs=None, headers=None):
if kwargs is None:
kwargs = {"fakearg": "fake"}
if headers is None:
headers = {}
scenario = utils.SwiftScenario(self.context)
scenario.generate_random_name = mock.MagicMock()
container = scenario._create_container(public=public,
headers=headers,
**kwargs)
self.assertEqual(container,
scenario.generate_random_name.return_value)
kwargs["headers"] = headers
kwargs["headers"]["X-Container-Read"] = ".r:*,.rlistings"
self.clients("swift").put_container.assert_called_once_with(
scenario.generate_random_name.return_value,
**kwargs)
self._test_atomic_action_timer(scenario.atomic_actions(),
"swift.create_container")
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["credential"])
sahara = clients.sahara()
if self.config["output_type"] == "swift":
swift = swift_utils.SwiftScenario(clients=clients,
context=self.context)
container_name = self.generate_random_name()
self.context["tenants"][tenant_id]["sahara"]["container"] = {
"name":
swift._create_container(container_name=container_name),
"output_swift_objects": []
}
self.setup_outputs_swift(swift, sahara, tenant_id,
container_name,
user["credential"].username,
user["credential"].password)
else:
self.setup_outputs_hdfs(sahara, tenant_id,
self.config["output_url_prefix"])