def _setup_for_existing_users(self): if (self.config["use_share_networks"] and not self.config["share_networks"]): msg = _("Usage of share networks was enabled but for deployment " "with existing users share networks also should be " "specified via arg 'share_networks'") raise exceptions.ContextSetupFailure(ctx_name=self.get_name(), msg=msg) # Set flag that says we will not delete/cleanup share networks self.context[CONTEXT_NAME]["delete_share_networks"] = False for tenant_name_or_id, share_networks in self.config[ "share_networks"].items(): # Verify project existence for tenant in self.context["tenants"].values(): if tenant_name_or_id in (tenant["id"], tenant["name"]): tenant_id = tenant["id"] existing_user = None for user in self.context["users"]: if user["tenant_id"] == tenant_id: existing_user = user break break else: msg = _("Provided tenant Name or ID '%s' was not found in " "existing tenants.") % tenant_name_or_id raise exceptions.ContextSetupFailure(ctx_name=self.get_name(), msg=msg) self.context["tenants"][tenant_id][CONTEXT_NAME] = {} self.context["tenants"][tenant_id][CONTEXT_NAME][ "share_networks"] = [] manila_scenario = manila_utils.ManilaScenario( {"user": existing_user}) existing_sns = manila_scenario._list_share_networks( detailed=False, search_opts={"project_id": tenant_id}) for sn_name_or_id in share_networks: # Verify share network existence for sn in existing_sns: if sn_name_or_id in (sn.id, sn.name): break else: msg = _("Specified share network '%(sn)s' does not " "exist for tenant '%(tenant_id)s'") % { "sn": sn_name_or_id, "tenant_id": tenant_id } raise exceptions.ContextSetupFailure( ctx_name=self.get_name(), msg=msg) # Set share network for project self.context["tenants"][tenant_id][CONTEXT_NAME][ "share_networks"].append(sn) # Add shared integer var per project that will be used as index # for list with share networks. It is required for balancing. self.context["tenants"][tenant_id][CONTEXT_NAME]["sn_iterator"] = ( utils.RAMInt())
def _run_scenario(self, cls, method_name, context, args): """Runs the specified scenario with given arguments. This method generates a constant load on the cloud under test by executing each scenario iteration using a pool of processes without pausing between iterations up to the number of times specified in the scenario config. :param cls: The Scenario class where the scenario is implemented :param method_name: Name of the method that implements the scenario :param context: context that contains users, admin & other information, that was created before scenario execution starts. :param args: Arguments to call the scenario method with :returns: List of results fore each single scenario iteration, where each result is a dictionary """ timeout = self.config.get("timeout", 0) # 0 means no timeout times = self.config.get("times", 1) concurrency = self.config.get("concurrency", 1) iteration_gen = utils.RAMInt() cpu_count = multiprocessing.cpu_count() max_cpu_used = min(cpu_count, self.config.get("max_cpu_count", cpu_count)) processes_to_start = min(max_cpu_used, times, concurrency) concurrency_per_worker, concurrency_overhead = divmod( concurrency, processes_to_start) self._log_debug_info(times=times, concurrency=concurrency, timeout=timeout, max_cpu_used=max_cpu_used, processes_to_start=processes_to_start, concurrency_per_worker=concurrency_per_worker, concurrency_overhead=concurrency_overhead) result_queue = multiprocessing.Queue() event_queue = multiprocessing.Queue() def worker_args_gen(concurrency_overhead): while True: yield (result_queue, iteration_gen, timeout, concurrency_per_worker + (concurrency_overhead and 1), times, context, cls, method_name, args, event_queue, self.aborted) if concurrency_overhead: concurrency_overhead -= 1 process_pool = self._create_process_pool( processes_to_start, _worker_process, worker_args_gen(concurrency_overhead)) self._join_processes(process_pool, result_queue, event_queue)
def _run_scenario(self, cls, method_name, context, args): """Runs the specified benchmark scenario with given arguments. Every single benchmark scenario iteration is executed with specified frequency (runs per second) in a pool of processes. The scenario will be launched for a fixed number of times in total (specified in the config). :param cls: The Scenario class where the scenario is implemented :param method_name: Name of the method that implements the scenario :param context: Benchmark context that contains users, admin & other information, that was created before benchmark started. :param args: Arguments to call the scenario method with :returns: List of results fore each single scenario iteration, where each result is a dictionary """ times = self.config["times"] timeout = self.config.get("timeout", 0) # 0 means no timeout iteration_gen = utils.RAMInt() cpu_count = multiprocessing.cpu_count() processes_to_start = min(cpu_count, times) rps_per_worker = float(self.config["rps"]) / processes_to_start times_per_worker, times_overhead = divmod(times, processes_to_start) self._log_debug_info(times=times, timeout=timeout, cpu_count=cpu_count, processes_to_start=processes_to_start, rps_per_worker=rps_per_worker, times_per_worker=times_per_worker, times_overhead=times_overhead) result_queue = multiprocessing.Queue() def worker_args_gen(times_overhead): while True: yield (result_queue, iteration_gen, timeout, rps_per_worker, times_per_worker + (times_overhead and 1), context, cls, method_name, args, self.aborted) if times_overhead: times_overhead -= 1 process_pool = self._create_process_pool( processes_to_start, _worker_process, worker_args_gen(times_overhead)) self._join_processes(process_pool, result_queue)
def test__next__(self, mock_multiprocessing): class MemInt(int): THRESHOLD = 5 def __iadd__(self, i): return MemInt((int(self) + i) % self.THRESHOLD) mock_lock = mock.MagicMock() mock_multiprocessing.Lock.return_value = mock_lock mock_multiprocessing.Value.return_value = mock.Mock(value=MemInt(0)) ram_int = utils.RAMInt() self.assertEqual(int(ram_int), 0) for i in range(MemInt.THRESHOLD - 1): self.assertEqual(ram_int.__next__(), i) self.assertRaises(StopIteration, ram_int.__next__) self.assertEqual(mock_lock.__enter__.mock_calls, [mock.call()] * MemInt.THRESHOLD) self.assertEqual(len(mock_lock.__exit__.mock_calls), MemInt.THRESHOLD)
def test__str__(self): self.assertEqual("0", str(utils.RAMInt())) self.assertEqual("20", str(utils.RAMInt(20)))
def test__int__(self): self.assertEqual(0, int(utils.RAMInt())) self.assertEqual(10, int(utils.RAMInt(10)))
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import random import netaddr import six from consts import ResourceType from rally.common import sshutils from rally.common import objects from rally.common import utils from rally.common import db cidr_incr = utils.RAMInt() ''' Find credential resource from DB by deployment uuid, and return info as a dict. :param deployment deployment uuid ''' def get_credential_from_resource(deployment): res = None if not isinstance(deployment, objects.Deployment): deployment = objects.Deployment.get(deployment) res = deployment.get_resources(type=ResourceType.CREDENTIAL)
def test_reset(self): ri = utils.RAMInt() ri.next() ri.reset() self.assertEqual(0, int(ri))
def test__iter__(self, mock_multiprocessing): ram_int = utils.RAMInt() self.assertEqual(iter(ram_int), ram_int)
def test__init__(self, mock_multiprocessing): utils.RAMInt() mock_multiprocessing.Lock.assert_called_once_with() mock_multiprocessing.Value.assert_called_once_with("I", 0)
def test__str__(self, mock_multiprocessing): mock_multiprocessing.Value.return_value = mock.Mock(value=42) self.assertEqual(str(utils.RAMInt()), "42")
import six from rally.common import cfg from rally.common import logging from rally.common import utils from rally import consts from rally import exceptions from neutronclient.common import exceptions as neutron_exceptions LOG = logging.getLogger(__name__) CONF = cfg.CONF cidr_incr = utils.RAMInt() ipv6_cidr_incr = utils.RAMInt() def generate_cidr(start_cidr="10.2.0.0/24"): """Generate next CIDR for network or subnet, without IP overlapping. This is process and thread safe, because `cidr_incr' points to value stored directly in RAM. This guarantees that CIDRs will be serial and unique even under hard multiprocessing/threading load. :param start_cidr: start CIDR str :returns: next available CIDR str """ if netaddr.IPNetwork(start_cidr).version == 4: cidr = str(netaddr.IPNetwork(start_cidr).next(next(cidr_incr)))
def test_next(self, mock_multi, mock_next): self.assertEqual(next(utils.RAMInt()), "next_value") mock_next.assert_called_once_with()
def test__int__(self, mock_multi): mock_multi.Value.return_value = mock.Mock(value=42) self.assertEqual(int(utils.RAMInt()), 42)
def test__next__(self): ri = utils.RAMInt() for i in range(0, 3): self.assertEqual(i, next(ri))
def test_next(self, mock_multiprocessing, mock_ram_int___next__): self.assertEqual(next(utils.RAMInt()), "next_value") mock_ram_int___next__.assert_called_once_with()
def test_next(self): ri = utils.RAMInt() for i in range(0, 3): self.assertEqual(i, ri.next())
def test_reset(self, mock_multiprocessing): ram_int = utils.RAMInt() self.assertRaises(TypeError, int, ram_int) ram_int.reset() self.assertEqual(int(ram_int), 0)
def _run_scenario(self, cls, method_name, context, args): """Runs the specified benchmark scenario with given arguments. Every single benchmark scenario iteration is executed with specified frequency (runs per second) in a pool of processes. The scenario will be launched for a fixed number of times in total (specified in the config). :param cls: The Scenario class where the scenario is implemented :param method_name: Name of the method that implements the scenario :param context: Benchmark context that contains users, admin & other information, that was created before benchmark started. :param args: Arguments to call the scenario method with :returns: List of results fore each single scenario iteration, where each result is a dictionary """ times = self.config["times"] timeout = self.config.get("timeout", 0) # 0 means no timeout iteration_gen = utils.RAMInt() cpu_count = multiprocessing.cpu_count() max_cpu_used = min(cpu_count, self.config.get("max_cpu_count", cpu_count)) def runs_per_second(rps_cfg, start_timer, number_of_processes): """At the given second return desired rps.""" if not isinstance(rps_cfg, dict): return float(rps_cfg) / number_of_processes stage_order = (time.time() - start_timer) / rps_cfg.get( "duration", 1) - 1 rps = (float(rps_cfg["start"] + rps_cfg["step"] * stage_order) / number_of_processes) return min(rps, float(rps_cfg["end"])) processes_to_start = min(max_cpu_used, times, self.config.get("max_concurrency", times)) times_per_worker, times_overhead = divmod(times, processes_to_start) # Determine concurrency per worker concurrency_per_worker, concurrency_overhead = divmod( self.config.get("max_concurrency", times), processes_to_start) self._log_debug_info(times=times, timeout=timeout, max_cpu_used=max_cpu_used, processes_to_start=processes_to_start, times_per_worker=times_per_worker, times_overhead=times_overhead, concurrency_per_worker=concurrency_per_worker, concurrency_overhead=concurrency_overhead) result_queue = multiprocessing.Queue() event_queue = multiprocessing.Queue() def worker_args_gen(times_overhead, concurrency_overhead): """Generate arguments for process worker. Remainder of threads per process division is distributed to process workers equally - one thread per each process worker until the remainder equals zero. The same logic is applied to concurrency overhead. :param times_overhead: remaining number of threads to be distributed to workers :param concurrency_overhead: remaining number of maximum concurrent threads to be distributed to workers """ while True: yield (result_queue, iteration_gen, timeout, times_per_worker + (times_overhead and 1), concurrency_per_worker + (concurrency_overhead and 1), context, cls, method_name, args, event_queue, self.aborted, runs_per_second, self.config["rps"], processes_to_start) if times_overhead: times_overhead -= 1 if concurrency_overhead: concurrency_overhead -= 1 process_pool = self._create_process_pool( processes_to_start, _worker_process, worker_args_gen(times_overhead, concurrency_overhead)) self._join_processes(process_pool, result_queue, event_queue)
# License for the specific language governing permissions and limitations # under the License. import netaddr from rally.common import logging from rally.common import utils LOG = logging.getLogger(__name__) _IPv4_START_CIDR = "10.2.0.0/24" _IPv6_START_CIDR = "dead:beaf::/64" _IPv4_CIDR_INCR = utils.RAMInt() _IPv6_CIDR_INCR = utils.RAMInt() def get_ip_version(ip): return netaddr.IPNetwork(ip).version def generate_cidr(ip_version=None, start_cidr=None): """Generate next CIDR for network or subnet, without IP overlapping. This is process and thread safe, because `cidr_incr' points to value stored directly in RAM. This guarantees that CIDRs will be serial and unique even under hard multiprocessing/threading load. :param ip_version: version of IP to take default value for start_cidr