Beispiel #1
0
def _worker_process(queue, iteration_gen, timeout, concurrency, times, context,
                    cls, method_name, args, aborted):
    """Start the scenario within threads.

    Spawn threads to support scenario execution for a fixed number of times.
    This generates a constant load on the cloud under test by executing each
    scenario iteration without pausing between iterations. Each thread runs
    the scenario method once with passed scenario arguments and context.
    After execution the result is appended to the queue.

    :param queue: queue object to append results
    :param iteration_gen: next iteration number generator
    :param timeout: operation's timeout
    :param concurrency: number of concurrently running scenario iterations
    :param times: total number of scenario iterations to be run
    :param context: scenario context object
    :param cls: scenario class
    :param method_name: scenario method name
    :param args: scenario args
    :param aborted: multiprocessing.Event that aborts load generation if
                    the flag is set
    """

    pool = collections.deque()
    alive_threads_in_pool = 0
    finished_threads_in_pool = 0

    base._log_worker_info(times=times,
                          concurrency=concurrency,
                          timeout=timeout,
                          cls=cls,
                          method_name=method_name,
                          args=args)

    iteration = next(iteration_gen)
    while iteration < times and not aborted.is_set():
        scenario_context = base._get_scenario_context(context)
        scenario_args = (iteration, cls, method_name, scenario_context, args)
        worker_args = (queue, scenario_args)

        thread = threading.Thread(target=base._worker_thread, args=worker_args)
        thread.start()
        pool.append((thread, time.time()))
        alive_threads_in_pool += 1

        while alive_threads_in_pool == concurrency:
            prev_finished_threads_in_pool = finished_threads_in_pool
            finished_threads_in_pool = 0
            for t in pool:
                if not t[0].isAlive():
                    finished_threads_in_pool += 1

            alive_threads_in_pool -= finished_threads_in_pool
            alive_threads_in_pool += prev_finished_threads_in_pool

            if alive_threads_in_pool < concurrency:
                # NOTE(boris-42): cleanup pool array. This is required because
                # in other case array length will be equal to times which
                # is unlimited big
                while pool and not pool[0][0].isAlive():
                    pool.popleft()[0].join()
                    finished_threads_in_pool -= 1
                break

            # we should wait to not create big noise with these checks
            time.sleep(0.001)
        iteration = next(iteration_gen)

    # Wait until all threads are done
    while pool:
        pool.popleft()[0].join()
Beispiel #2
0
def _worker_process(queue, iteration_gen, timeout, concurrency, times, context,
                    cls, method_name, args, aborted):
    """Start the scenario within threads.

    Spawn threads to support scenario execution for a fixed number of times.
    This generates a constant load on the cloud under test by executing each
    scenario iteration without pausing between iterations. Each thread runs
    the scenario method once with passed scenario arguments and context.
    After execution the result is appended to the queue.

    :param queue: queue object to append results
    :param iteration_gen: next iteration number generator
    :param timeout: operation's timeout
    :param concurrency: number of concurrently running scenario iterations
    :param times: total number of scenario iterations to be run
    :param context: scenario context object
    :param cls: scenario class
    :param method_name: scenario method name
    :param args: scenario args
    :param aborted: multiprocessing.Event that aborts load generation if
                    the flag is set
    """

    pool = collections.deque()
    alive_threads_in_pool = 0
    finished_threads_in_pool = 0

    base._log_worker_info(times=times, concurrency=concurrency,
                          timeout=timeout, cls=cls, method_name=method_name,
                          args=args)

    iteration = next(iteration_gen)
    while iteration < times and not aborted.is_set():
        scenario_context = base._get_scenario_context(context)
        scenario_args = (iteration, cls, method_name, scenario_context, args)
        worker_args = (queue, scenario_args)

        thread = threading.Thread(target=base._worker_thread,
                                  args=worker_args)
        thread.start()
        pool.append((thread, time.time()))
        alive_threads_in_pool += 1

        while alive_threads_in_pool == concurrency:
            prev_finished_threads_in_pool = finished_threads_in_pool
            finished_threads_in_pool = 0
            for t in pool:
                if not t[0].isAlive():
                    finished_threads_in_pool += 1

            alive_threads_in_pool -= finished_threads_in_pool
            alive_threads_in_pool += prev_finished_threads_in_pool

            if alive_threads_in_pool < concurrency:
                # NOTE(boris-42): cleanup pool array. This is required because
                # in other case array length will be equal to times which
                # is unlimited big
                while pool and not pool[0][0].isAlive():
                    pool.popleft()[0].join()
                    finished_threads_in_pool -= 1
                break

            # we should wait to not create big noise with these checks
            time.sleep(0.001)
        iteration = next(iteration_gen)

    # Wait until all threads are done
    while pool:
        pool.popleft()[0].join()
Beispiel #3
0
def _worker_process(queue, iteration_gen, timeout, rps, times, context, cls,
                    method_name, args, aborted):
    """Start scenario within threads.

    Spawn N threads per second. Each thread runs the scenario once, and appends
    the result to the queue.

    :param queue: queue object to append results
    :param iteration_gen: next iteration number generator
    :param timeout: operation's timeout
    :param rps: number of scenario iterations to be run per one second
    :param times: total number of scenario iterations to be run
    :param context: scenario context object
    :param cls: scenario class
    :param method_name: scenario method name
    :param args: scenario args
    :param aborted: multiprocessing.Event that aborts load generation if
                    the flag is set
    """

    pool = collections.deque()
    start = time.time()
    sleep = 1.0 / rps

    base._log_worker_info(times=times,
                          rps=rps,
                          timeout=timeout,
                          cls=cls,
                          method_name=method_name,
                          args=args)

    # Injecting timeout to exclude situations, where start time and
    # actual time are neglible close

    randsleep_delay = random.randint(int(sleep / 2 * 100), int(sleep * 100))
    time.sleep(randsleep_delay / 100.0)

    i = 0
    while i < times and not aborted.is_set():
        scenario_context = base._get_scenario_context(context)
        scenario_args = (next(iteration_gen), cls, method_name,
                         scenario_context, args)
        worker_args = (queue, scenario_args)
        thread = threading.Thread(target=base._worker_thread, args=worker_args)
        i += 1
        thread.start()
        pool.append(thread)

        time_gap = time.time() - start
        real_rps = i / time_gap if time_gap else "Infinity"

        LOG.debug("Worker: %s rps: %s (requested rps: %s)" %
                  (i, real_rps, rps))

        # try to join latest thread(s) until it finished, or until time to
        # start new thread
        while i / (time.time() - start) > rps:
            if pool:
                pool[0].join(sleep)
                if not pool[0].isAlive():
                    pool.popleft()
            else:
                time.sleep(sleep)

    while pool:
        thr = pool.popleft()
        thr.join()
Beispiel #4
0
def _worker_process(queue, iteration_gen, timeout, rps, times,
                    max_concurrent, context, cls, method_name,
                    args, aborted):
    """Start scenario within threads.

    Spawn N threads per second. Each thread runs the scenario once, and appends
    result to queue. A maximum of max_concurrent threads will be ran
    concurrently.

    :param queue: queue object to append results
    :param iteration_gen: next iteration number generator
    :param timeout: operation's timeout
    :param rps: number of scenario iterations to be run per one second
    :param times: total number of scenario iterations to be run
    :param max_concurrent: maximum worker concurrency
    :param context: scenario context object
    :param cls: scenario class
    :param method_name: scenario method name
    :param args: scenario args
    :param aborted: multiprocessing.Event that aborts load generation if
                    the flag is set
    """

    pool = collections.deque()
    start = time.time()
    sleep = 1.0 / rps

    base._log_worker_info(times=times, rps=rps, timeout=timeout,
                          cls=cls, method_name=method_name, args=args)

    # Injecting timeout to exclude situations, where start time and
    # actual time are negligible close

    randsleep_delay = random.randint(int(sleep / 2 * 100), int(sleep * 100))
    time.sleep(randsleep_delay / 100.0)

    i = 0
    while i < times and not aborted.is_set():
        scenario_context = base._get_scenario_context(context)
        scenario_args = (next(iteration_gen), cls, method_name,
                         scenario_context, args)
        worker_args = (queue, scenario_args)
        thread = threading.Thread(target=base._worker_thread,
                                  args=worker_args)
        i += 1
        thread.start()
        pool.append(thread)

        time_gap = time.time() - start
        real_rps = i / time_gap if time_gap else "Infinity"

        LOG.debug("Worker: %s rps: %s (requested rps: %s)" %
                  (i, real_rps, rps))

        # try to join latest thread(s) until it finished, or until time to
        # start new thread (if we have concurrent slots available)
        while i / (time.time() - start) > rps or len(pool) >= max_concurrent:
            if pool:
                pool[0].join(sleep)
                if not pool[0].isAlive():
                    pool.popleft()
            else:
                time.sleep(sleep)

    while pool:
        thr = pool.popleft()
        thr.join()