Exemplo n.º 1
0
def main():
    '''To run the static content load test, make sure you've bootstrapped your
       test environment and set up your config.yml file, then run the following:
          S3TEST_CONF=config.yml virtualenv/bin/python generate_objects.py -O urls.txt --seed 1234

        This creates a bucket with your S3 credentials (from config.yml) and
        fills it with garbage objects as described in generate_objects.conf.
        It writes a list of URLS to those objects to ./urls.txt.

        Once you have objcts in your bucket, run the siege benchmarking program:
            siege --rc ./siege.conf -r 5

        This tells siege to read the ./siege.conf config file which tells it to
        use the urls in ./urls.txt and log to ./siege.log. It hits each url in
        urls.txt 5 times (-r flag).

        Results are printed to the terminal and written in CSV format to
        ./siege.log
    '''
    (options, args) = parse_opts()

    #SETUP
    random.seed(options.seed if options.seed else None)
    conn = common.s3.main

    if options.outfile:
        OUTFILE = open(options.outfile, 'w')
    elif common.config.file_generation.url_file:
        OUTFILE = open(common.config.file_generation.url_file, 'w')
    else:
        OUTFILE = sys.stdout

    if options.bucket:
        bucket = conn.create_bucket(options.bucket)
    else:
        bucket = common.get_new_bucket()

    keys = []
    print >> OUTFILE, 'bucket: %s' % bucket.name
    print >> sys.stderr, 'setup complete, generating files'
    for profile in common.config.file_generation.groups:
        seed = random.random()
        files = get_random_files(profile[0], profile[1], profile[2], seed)
        keys += upload_objects(bucket, files, seed)

    print >> sys.stderr, 'finished sending files. generating urls'
    for key in keys:
        print >> OUTFILE, key.generate_url(30758400) #valid for 1 year

    print >> sys.stderr, 'done'
Exemplo n.º 2
0
def main():
    '''To run the static content load test, make sure you've bootstrapped your
       test environment and set up your config.yml file, then run the following:
          S3TEST_CONF=config.yml virtualenv/bin/python generate_objects.py -O urls.txt --seed 1234

        This creates a bucket with your S3 credentials (from config.yml) and
        fills it with garbage objects as described in generate_objects.conf.
        It writes a list of URLS to those objects to ./urls.txt.

        Once you have objcts in your bucket, run the siege benchmarking program:
            siege --rc ./siege.conf -r 5

        This tells siege to read the ./siege.conf config file which tells it to
        use the urls in ./urls.txt and log to ./siege.log. It hits each url in
        urls.txt 5 times (-r flag).

        Results are printed to the terminal and written in CSV format to
        ./siege.log
    '''
    (options, args) = parse_opts()

    #SETUP
    random.seed(options.seed if options.seed else None)
    conn = common.s3.main

    if options.outfile:
        OUTFILE = open(options.outfile, 'w')
    elif common.config.file_generation.url_file:
        OUTFILE = open(common.config.file_generation.url_file, 'w')
    else:
        OUTFILE = sys.stdout

    if options.bucket:
        bucket = conn.create_bucket(options.bucket)
    else:
        bucket = common.get_new_bucket()

    keys = []
    print >> OUTFILE, 'bucket: %s' % bucket.name
    print >> sys.stderr, 'setup complete, generating files'
    for profile in common.config.file_generation.groups:
        seed = random.random()
        files = get_random_files(profile[0], profile[1], profile[2], seed)
        keys += upload_objects(bucket, files, seed)

    print >> sys.stderr, 'finished sending files. generating urls'
    for key in keys:
        print >> OUTFILE, key.generate_url(30758400)  #valid for 1 year

    print >> sys.stderr, 'done'
Exemplo n.º 3
0
def main():
    # parse options
    (options, args) = parse_options()

    try:
        # setup
        common.setup()
        bucket = common.get_new_bucket()
        print "Created bucket: {name}".format(name=bucket.name)
        r = None
        if (options.rewrite):
            r = random.randint(0, 65535)
        q = gevent.queue.Queue()

        # main work
        print "Using file size: {size} +- {stddev}".format(
            size=options.file_size, stddev=options.stddev)
        print "Spawning {r} readers and {w} writers...".format(
            r=options.num_readers, w=options.num_writers)
        greenlets = []
        greenlets += [
            gevent.spawn(writer,
                         options.duration,
                         bucket,
                         name=x,
                         queue=q,
                         file_size=options.file_size,
                         file_stddev=options.stddev,
                         quantity=options.quantity,
                         file_name_seed=r) for x in xrange(options.num_writers)
        ]
        greenlets += [
            gevent.spawn(reader, options.duration, bucket, name=x, queue=q)
            for x in xrange(options.num_readers)
        ]
        gevent.spawn_later(options.duration, lambda: q.put(StopIteration))

        total_read = 0
        total_write = 0
        read_success = 0
        read_failure = 0
        write_success = 0
        write_failure = 0
        for item in q:
            print item
            if item.type == Result.TYPE_READER:
                if item.success:
                    read_success += 1
                    total_read += item.size
                else:
                    read_failure += 1
            elif item.type == Result.TYPE_WRITER:
                if item.success:
                    write_success += 1
                    total_write += item.size
                else:
                    write_failure += 1

        # overall stats
        print "--- Stats ---"
        print "Total Read:  {read} MB ({mbps} MB/s)".format(
            read=(total_read / 1024.0),
            mbps=(total_read / 1024.0 / options.duration))
        print "Total Write: {write} MB ({mbps} MB/s)".format(
            write=(total_write / 1024.0),
            mbps=(total_write / 1024.0 / options.duration))
        print "Read filures: {num} ({percent}%)".format(
            num=read_failure,
            percent=(100.0 * read_failure /
                     max(read_failure + read_success, 1)))
        print "Write failures: {num} ({percent}%)".format(
            num=write_failure,
            percent=(100.0 * write_failure /
                     max(write_failure + write_success, 1)))

        gevent.joinall(greenlets, timeout=1)
    except Exception as e:
        print e
    finally:
        # cleanup
        if options.cleanup:
            common.teardown()
Exemplo n.º 4
0
def main():
    # parse options
    (options, args) = parse_options()

    try:
        # setup
        common.setup()
        bucket = common.get_new_bucket()
        print "Created bucket: {name}".format(name=bucket.name)
        r = None
        if (options.rewrite):
            r = random.randint(0, 65535)
        q = gevent.queue.Queue()

        # main work
        print "Using file size: {size} +- {stddev}".format(size=options.file_size, stddev=options.stddev)
        print "Spawning {r} readers and {w} writers...".format(r=options.num_readers, w=options.num_writers)
        greenlets = []
        greenlets += [gevent.spawn(writer, options.duration, bucket,
            name=x,
            queue=q,
            file_size=options.file_size,
            file_stddev=options.stddev,
            quantity=options.quantity,
            file_name_seed=r
            ) for x in xrange(options.num_writers)]
        greenlets += [gevent.spawn(reader, options.duration, bucket,
                name=x,
                queue=q
                ) for x in xrange(options.num_readers)]
        gevent.spawn_later(options.duration, lambda: q.put(StopIteration))

        total_read = 0
        total_write = 0
        read_success = 0
        read_failure = 0
        write_success = 0
        write_failure = 0
        for item in q:
            print item
            if item.type == Result.TYPE_READER:
                if item.success:
                    read_success += 1
                    total_read += item.size
                else:
                    read_failure += 1
            elif item.type == Result.TYPE_WRITER:
                if item.success:
                    write_success += 1
                    total_write += item.size
                else:
                    write_failure += 1

        # overall stats
        print "--- Stats ---"
        print "Total Read:  {read} MB ({mbps} MB/s)".format(
            read=(total_read/1024.0),
            mbps=(total_read/1024.0/options.duration)
            )
        print "Total Write: {write} MB ({mbps} MB/s)".format(
            write=(total_write/1024.0),
            mbps=(total_write/1024.0/options.duration)
            )
        print "Read filures: {num} ({percent}%)".format(
            num=read_failure,
            percent=(100.0*read_failure/max(read_failure+read_success, 1))
            )
        print "Write failures: {num} ({percent}%)".format(
            num=write_failure,
            percent=(100.0*write_failure/max(write_failure+write_success, 1))
            )

        gevent.joinall(greenlets, timeout=1)
    except Exception as e:
        print e
    finally:
        # cleanup
        if options.cleanup:
            common.teardown()