示例#1
0
    def run(self, client, metrics):
        # use client.set + client.flush_pipeline pattern
        client.pipeline_mode = True

        stats = client.get_stats()
        capacity = stats['limit_maxbytes']
        capacity_fmt = insert_number_commas(capacity)

        metrics.pct_full = self.get_pct_full(client)
        metrics.batch_size = 100
        metrics.bytes_cum = 0
        metrics.time_cum = 0
        metrics.time_total_cum = 0
        metrics.items_cum = 0
        rate = -1

        time_total_st = time.time()

        while metrics.pct_full < self.percentage:
            self.write("Cache is %.2f%% full of %s, inserting %s items (rate: %s items/s)" %
                       (metrics.pct_full, capacity_fmt, metrics.batch_size,
                        insert_number_commas(str(int(rate)))))

            # Pre-generate keys and values to avoid timing this work
            # TODO allow tuning the sizes of keys and values
            keys = [generate_random_key_uuid(10)
                    for _ in xrange(metrics.batch_size)]
            values = [generate_random_data(100, 1000)
                      for _ in xrange(metrics.batch_size)]

            for key, value in izip(keys, values):
                if not self._runnable:
                    return

                client.set(key, value, noreply=True)

                metrics.bytes_cum += len(key) + len(value)

            time_st = time.time()
            client.flush_pipeline()
            duration = time.time() - time_st

            rate = metrics.batch_size / duration
            metrics.time_cum += duration
            metrics.items_cum += metrics.batch_size

            metrics.pct_full = self.get_pct_full(client)
            metrics.time_total_cum = time.time() - time_total_st
示例#2
0
    def pre_tasklets(self, client, state):
        stats = client.get_stats()
        capacity = stats['limit_maxbytes']
        capacity_fmt = insert_number_commas(capacity)

        state.time_start = time.time()
        self.write("Filling to ~%s%% (of %s)" % (self.percentage, capacity_fmt))
示例#3
0
    def post_tasklets(self, client, state, metrics_list):
        state.time_stop = time.time()
        state.duration = state.time_stop - state.time_start

        time_cum = sum([m.time_cum for m in metrics_list])
        time_total_cum = sum([m.time_total_cum for m in metrics_list])
        overhead_pct = 100 * (time_total_cum - time_cum) / time_total_cum if time_total_cum else 0

        items_cum = sum([m.items_cum for m in metrics_list])
        items_cum_str = insert_number_commas(str(items_cum))
        rate_items_cum = float(items_cum) / float(time_cum) if time_cum > 0 else 0
        rate_items_str = insert_number_commas(str(int(rate_items_cum)))

        bytes_cum = sum([m.bytes_cum for m in metrics_list])
        bytes_cum_str = insert_number_commas(str(bytes_cum))
        rate_bytes_cum = float(bytes_cum) / float(time_cum) if time_cum > 0 else 0
        rate_bytes_str = insert_number_commas(str(int(rate_bytes_cum)))

        self.write("Done filling, took %.2fs to insert %s items"
                   " (net avg rate: %s items/s - %s bytes/s)" %
                   (state.duration, items_cum_str, rate_items_str, rate_bytes_str))
        self.write("Spent %.2fs in network io, %.2fs in total (%.2f%% overhead - data gen, thread scheduling)"
                   % (time_cum, time_total_cum, overhead_pct))
        self.write("Wrote %s bytes in total" % bytes_cum_str)