def _setup_for_existing_users(self):
if (self.config["use_share_networks"]
and not self.config["share_networks"]):
msg = _("Usage of share networks was enabled but for deployment "
"with existing users share networks also should be "
"specified via arg 'share_networks'")
raise exceptions.ContextSetupFailure(ctx_name=self.get_name(),
msg=msg)
# Set flag that says we will not delete/cleanup share networks
self.context[CONTEXT_NAME]["delete_share_networks"] = False
for tenant_name_or_id, share_networks in self.config[
"share_networks"].items():
# Verify project existence
for tenant in self.context["tenants"].values():
if tenant_name_or_id in (tenant["id"], tenant["name"]):
tenant_id = tenant["id"]
existing_user = None
for user in self.context["users"]:
if user["tenant_id"] == tenant_id:
existing_user = user
break
break
else:
msg = _("Provided tenant Name or ID '%s' was not found in "
"existing tenants.") % tenant_name_or_id
raise exceptions.ContextSetupFailure(ctx_name=self.get_name(),
msg=msg)
self.context["tenants"][tenant_id][CONTEXT_NAME] = {}
self.context["tenants"][tenant_id][CONTEXT_NAME][
"share_networks"] = []
manila_scenario = manila_utils.ManilaScenario(
{"user": existing_user})
existing_sns = manila_scenario._list_share_networks(
detailed=False, search_opts={"project_id": tenant_id})
for sn_name_or_id in share_networks:
# Verify share network existence
for sn in existing_sns:
if sn_name_or_id in (sn.id, sn.name):
break
else:
msg = _("Specified share network '%(sn)s' does not "
"exist for tenant '%(tenant_id)s'") % {
"sn": sn_name_or_id,
"tenant_id": tenant_id
}
raise exceptions.ContextSetupFailure(
ctx_name=self.get_name(), msg=msg)
# Set share network for project
self.context["tenants"][tenant_id][CONTEXT_NAME][
"share_networks"].append(sn)
# Add shared integer var per project that will be used as index
# for list with share networks. It is required for balancing.
self.context["tenants"][tenant_id][CONTEXT_NAME]["sn_iterator"] = (
utils.RAMInt())
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified scenario with given arguments.
This method generates a constant load on the cloud under test by
executing each scenario iteration using a pool of processes without
pausing between iterations up to the number of times specified
in the scenario config.
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: context that contains users, admin & other
information, that was created before scenario
execution starts.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
timeout = self.config.get("timeout", 0) # 0 means no timeout
times = self.config.get("times", 1)
concurrency = self.config.get("concurrency", 1)
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
max_cpu_used = min(cpu_count,
self.config.get("max_cpu_count", cpu_count))
processes_to_start = min(max_cpu_used, times, concurrency)
concurrency_per_worker, concurrency_overhead = divmod(
concurrency, processes_to_start)
self._log_debug_info(times=times,
concurrency=concurrency,
timeout=timeout,
max_cpu_used=max_cpu_used,
processes_to_start=processes_to_start,
concurrency_per_worker=concurrency_per_worker,
concurrency_overhead=concurrency_overhead)
result_queue = multiprocessing.Queue()
event_queue = multiprocessing.Queue()
def worker_args_gen(concurrency_overhead):
while True:
yield (result_queue, iteration_gen, timeout,
concurrency_per_worker + (concurrency_overhead and 1),
times, context, cls, method_name, args, event_queue,
self.aborted)
if concurrency_overhead:
concurrency_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(concurrency_overhead))
self._join_processes(process_pool, result_queue, event_queue)
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified benchmark scenario with given arguments.
Every single benchmark scenario iteration is executed with specified
frequency (runs per second) in a pool of processes. The scenario will
be launched for a fixed number of times in total (specified in the
config).
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: Benchmark context that contains users, admin & other
information, that was created before benchmark started.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
times = self.config["times"]
timeout = self.config.get("timeout", 0) # 0 means no timeout
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
processes_to_start = min(cpu_count, times)
rps_per_worker = float(self.config["rps"]) / processes_to_start
times_per_worker, times_overhead = divmod(times, processes_to_start)
self._log_debug_info(times=times,
timeout=timeout,
cpu_count=cpu_count,
processes_to_start=processes_to_start,
rps_per_worker=rps_per_worker,
times_per_worker=times_per_worker,
times_overhead=times_overhead)
result_queue = multiprocessing.Queue()
def worker_args_gen(times_overhead):
while True:
yield (result_queue, iteration_gen, timeout, rps_per_worker,
times_per_worker + (times_overhead and 1), context, cls,
method_name, args, self.aborted)
if times_overhead:
times_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(times_overhead))
self._join_processes(process_pool, result_queue)
def test__next__(self, mock_multiprocessing):
class MemInt(int):
THRESHOLD = 5
def __iadd__(self, i):
return MemInt((int(self) + i) % self.THRESHOLD)
mock_lock = mock.MagicMock()
mock_multiprocessing.Lock.return_value = mock_lock
mock_multiprocessing.Value.return_value = mock.Mock(value=MemInt(0))
ram_int = utils.RAMInt()
self.assertEqual(int(ram_int), 0)
for i in range(MemInt.THRESHOLD - 1):
self.assertEqual(ram_int.__next__(), i)
self.assertRaises(StopIteration, ram_int.__next__)
self.assertEqual(mock_lock.__enter__.mock_calls,
[mock.call()] * MemInt.THRESHOLD)
self.assertEqual(len(mock_lock.__exit__.mock_calls), MemInt.THRESHOLD)
def test__str__(self):
self.assertEqual("0", str(utils.RAMInt()))
self.assertEqual("20", str(utils.RAMInt(20)))
def test__int__(self):
self.assertEqual(0, int(utils.RAMInt()))
self.assertEqual(10, int(utils.RAMInt(10)))
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import random
import netaddr
import six
from consts import ResourceType
from rally.common import sshutils
from rally.common import objects
from rally.common import utils
from rally.common import db
cidr_incr = utils.RAMInt()
'''
Find credential resource from DB by deployment uuid, and return
info as a dict.
:param deployment deployment uuid
'''
def get_credential_from_resource(deployment):
res = None
if not isinstance(deployment, objects.Deployment):
deployment = objects.Deployment.get(deployment)
res = deployment.get_resources(type=ResourceType.CREDENTIAL)
def test_reset(self):
ri = utils.RAMInt()
ri.next()
ri.reset()
self.assertEqual(0, int(ri))
def test__iter__(self, mock_multiprocessing):
ram_int = utils.RAMInt()
self.assertEqual(iter(ram_int), ram_int)
def test__init__(self, mock_multiprocessing):
utils.RAMInt()
mock_multiprocessing.Lock.assert_called_once_with()
mock_multiprocessing.Value.assert_called_once_with("I", 0)
def test__str__(self, mock_multiprocessing):
mock_multiprocessing.Value.return_value = mock.Mock(value=42)
self.assertEqual(str(utils.RAMInt()), "42")
import six
from rally.common import cfg
from rally.common import logging
from rally.common import utils
from rally import consts
from rally import exceptions
from neutronclient.common import exceptions as neutron_exceptions
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
cidr_incr = utils.RAMInt()
ipv6_cidr_incr = utils.RAMInt()
def generate_cidr(start_cidr="10.2.0.0/24"):
"""Generate next CIDR for network or subnet, without IP overlapping.
This is process and thread safe, because `cidr_incr' points to
value stored directly in RAM. This guarantees that CIDRs will be
serial and unique even under hard multiprocessing/threading load.
:param start_cidr: start CIDR str
:returns: next available CIDR str
"""
if netaddr.IPNetwork(start_cidr).version == 4:
cidr = str(netaddr.IPNetwork(start_cidr).next(next(cidr_incr)))
def test_next(self, mock_multi, mock_next):
self.assertEqual(next(utils.RAMInt()), "next_value")
mock_next.assert_called_once_with()
def test__int__(self, mock_multi):
mock_multi.Value.return_value = mock.Mock(value=42)
self.assertEqual(int(utils.RAMInt()), 42)
def test__next__(self):
ri = utils.RAMInt()
for i in range(0, 3):
self.assertEqual(i, next(ri))
def test_next(self, mock_multiprocessing, mock_ram_int___next__):
self.assertEqual(next(utils.RAMInt()), "next_value")
mock_ram_int___next__.assert_called_once_with()
def test_next(self):
ri = utils.RAMInt()
for i in range(0, 3):
self.assertEqual(i, ri.next())
def test_reset(self, mock_multiprocessing):
ram_int = utils.RAMInt()
self.assertRaises(TypeError, int, ram_int)
ram_int.reset()
self.assertEqual(int(ram_int), 0)
def _run_scenario(self, cls, method_name, context, args):
"""Runs the specified benchmark scenario with given arguments.
Every single benchmark scenario iteration is executed with specified
frequency (runs per second) in a pool of processes. The scenario will
be launched for a fixed number of times in total (specified in the
config).
:param cls: The Scenario class where the scenario is implemented
:param method_name: Name of the method that implements the scenario
:param context: Benchmark context that contains users, admin & other
information, that was created before benchmark started.
:param args: Arguments to call the scenario method with
:returns: List of results fore each single scenario iteration,
where each result is a dictionary
"""
times = self.config["times"]
timeout = self.config.get("timeout", 0) # 0 means no timeout
iteration_gen = utils.RAMInt()
cpu_count = multiprocessing.cpu_count()
max_cpu_used = min(cpu_count,
self.config.get("max_cpu_count", cpu_count))
def runs_per_second(rps_cfg, start_timer, number_of_processes):
"""At the given second return desired rps."""
if not isinstance(rps_cfg, dict):
return float(rps_cfg) / number_of_processes
stage_order = (time.time() - start_timer) / rps_cfg.get(
"duration", 1) - 1
rps = (float(rps_cfg["start"] + rps_cfg["step"] * stage_order) /
number_of_processes)
return min(rps, float(rps_cfg["end"]))
processes_to_start = min(max_cpu_used, times,
self.config.get("max_concurrency", times))
times_per_worker, times_overhead = divmod(times, processes_to_start)
# Determine concurrency per worker
concurrency_per_worker, concurrency_overhead = divmod(
self.config.get("max_concurrency", times), processes_to_start)
self._log_debug_info(times=times,
timeout=timeout,
max_cpu_used=max_cpu_used,
processes_to_start=processes_to_start,
times_per_worker=times_per_worker,
times_overhead=times_overhead,
concurrency_per_worker=concurrency_per_worker,
concurrency_overhead=concurrency_overhead)
result_queue = multiprocessing.Queue()
event_queue = multiprocessing.Queue()
def worker_args_gen(times_overhead, concurrency_overhead):
"""Generate arguments for process worker.
Remainder of threads per process division is distributed to
process workers equally - one thread per each process worker
until the remainder equals zero. The same logic is applied
to concurrency overhead.
:param times_overhead: remaining number of threads to be
distributed to workers
:param concurrency_overhead: remaining number of maximum
concurrent threads to be
distributed to workers
"""
while True:
yield (result_queue, iteration_gen, timeout,
times_per_worker + (times_overhead and 1),
concurrency_per_worker + (concurrency_overhead and 1),
context, cls, method_name, args, event_queue,
self.aborted, runs_per_second, self.config["rps"],
processes_to_start)
if times_overhead:
times_overhead -= 1
if concurrency_overhead:
concurrency_overhead -= 1
process_pool = self._create_process_pool(
processes_to_start, _worker_process,
worker_args_gen(times_overhead, concurrency_overhead))
self._join_processes(process_pool, result_queue, event_queue)
# License for the specific language governing permissions and limitations
# under the License.
import netaddr
from rally.common import logging
from rally.common import utils
LOG = logging.getLogger(__name__)
_IPv4_START_CIDR = "10.2.0.0/24"
_IPv6_START_CIDR = "dead:beaf::/64"
_IPv4_CIDR_INCR = utils.RAMInt()
_IPv6_CIDR_INCR = utils.RAMInt()
def get_ip_version(ip):
return netaddr.IPNetwork(ip).version
def generate_cidr(ip_version=None, start_cidr=None):
"""Generate next CIDR for network or subnet, without IP overlapping.
This is process and thread safe, because `cidr_incr' points to
value stored directly in RAM. This guarantees that CIDRs will be
serial and unique even under hard multiprocessing/threading load.
:param ip_version: version of IP to take default value for start_cidr