def main(): '''To run the static content load test, make sure you've bootstrapped your test environment and set up your config.yml file, then run the following: S3TEST_CONF=config.yml virtualenv/bin/python generate_objects.py -O urls.txt --seed 1234 This creates a bucket with your S3 credentials (from config.yml) and fills it with garbage objects as described in generate_objects.conf. It writes a list of URLS to those objects to ./urls.txt. Once you have objcts in your bucket, run the siege benchmarking program: siege --rc ./siege.conf -r 5 This tells siege to read the ./siege.conf config file which tells it to use the urls in ./urls.txt and log to ./siege.log. It hits each url in urls.txt 5 times (-r flag). Results are printed to the terminal and written in CSV format to ./siege.log ''' (options, args) = parse_opts() #SETUP random.seed(options.seed if options.seed else None) conn = common.s3.main if options.outfile: OUTFILE = open(options.outfile, 'w') elif common.config.file_generation.url_file: OUTFILE = open(common.config.file_generation.url_file, 'w') else: OUTFILE = sys.stdout if options.bucket: bucket = conn.create_bucket(options.bucket) else: bucket = common.get_new_bucket() keys = [] print >> OUTFILE, 'bucket: %s' % bucket.name print >> sys.stderr, 'setup complete, generating files' for profile in common.config.file_generation.groups: seed = random.random() files = get_random_files(profile[0], profile[1], profile[2], seed) keys += upload_objects(bucket, files, seed) print >> sys.stderr, 'finished sending files. generating urls' for key in keys: print >> OUTFILE, key.generate_url(30758400) #valid for 1 year print >> sys.stderr, 'done'
def main(): # parse options (options, args) = parse_options() try: # setup common.setup() bucket = common.get_new_bucket() print "Created bucket: {name}".format(name=bucket.name) r = None if (options.rewrite): r = random.randint(0, 65535) q = gevent.queue.Queue() # main work print "Using file size: {size} +- {stddev}".format( size=options.file_size, stddev=options.stddev) print "Spawning {r} readers and {w} writers...".format( r=options.num_readers, w=options.num_writers) greenlets = [] greenlets += [ gevent.spawn(writer, options.duration, bucket, name=x, queue=q, file_size=options.file_size, file_stddev=options.stddev, quantity=options.quantity, file_name_seed=r) for x in xrange(options.num_writers) ] greenlets += [ gevent.spawn(reader, options.duration, bucket, name=x, queue=q) for x in xrange(options.num_readers) ] gevent.spawn_later(options.duration, lambda: q.put(StopIteration)) total_read = 0 total_write = 0 read_success = 0 read_failure = 0 write_success = 0 write_failure = 0 for item in q: print item if item.type == Result.TYPE_READER: if item.success: read_success += 1 total_read += item.size else: read_failure += 1 elif item.type == Result.TYPE_WRITER: if item.success: write_success += 1 total_write += item.size else: write_failure += 1 # overall stats print "--- Stats ---" print "Total Read: {read} MB ({mbps} MB/s)".format( read=(total_read / 1024.0), mbps=(total_read / 1024.0 / options.duration)) print "Total Write: {write} MB ({mbps} MB/s)".format( write=(total_write / 1024.0), mbps=(total_write / 1024.0 / options.duration)) print "Read filures: {num} ({percent}%)".format( num=read_failure, percent=(100.0 * read_failure / max(read_failure + read_success, 1))) print "Write failures: {num} ({percent}%)".format( num=write_failure, percent=(100.0 * write_failure / max(write_failure + write_success, 1))) gevent.joinall(greenlets, timeout=1) except Exception as e: print e finally: # cleanup if options.cleanup: common.teardown()
def main(): # parse options (options, args) = parse_options() try: # setup common.setup() bucket = common.get_new_bucket() print "Created bucket: {name}".format(name=bucket.name) r = None if (options.rewrite): r = random.randint(0, 65535) q = gevent.queue.Queue() # main work print "Using file size: {size} +- {stddev}".format(size=options.file_size, stddev=options.stddev) print "Spawning {r} readers and {w} writers...".format(r=options.num_readers, w=options.num_writers) greenlets = [] greenlets += [gevent.spawn(writer, options.duration, bucket, name=x, queue=q, file_size=options.file_size, file_stddev=options.stddev, quantity=options.quantity, file_name_seed=r ) for x in xrange(options.num_writers)] greenlets += [gevent.spawn(reader, options.duration, bucket, name=x, queue=q ) for x in xrange(options.num_readers)] gevent.spawn_later(options.duration, lambda: q.put(StopIteration)) total_read = 0 total_write = 0 read_success = 0 read_failure = 0 write_success = 0 write_failure = 0 for item in q: print item if item.type == Result.TYPE_READER: if item.success: read_success += 1 total_read += item.size else: read_failure += 1 elif item.type == Result.TYPE_WRITER: if item.success: write_success += 1 total_write += item.size else: write_failure += 1 # overall stats print "--- Stats ---" print "Total Read: {read} MB ({mbps} MB/s)".format( read=(total_read/1024.0), mbps=(total_read/1024.0/options.duration) ) print "Total Write: {write} MB ({mbps} MB/s)".format( write=(total_write/1024.0), mbps=(total_write/1024.0/options.duration) ) print "Read filures: {num} ({percent}%)".format( num=read_failure, percent=(100.0*read_failure/max(read_failure+read_success, 1)) ) print "Write failures: {num} ({percent}%)".format( num=write_failure, percent=(100.0*write_failure/max(write_failure+write_success, 1)) ) gevent.joinall(greenlets, timeout=1) except Exception as e: print e finally: # cleanup if options.cleanup: common.teardown()