def build(benchmarker_config, test_names, build_log_dir=os.devnull): ''' Builds the test docker containers ''' tests = gather_tests(include=test_names, benchmarker_config=benchmarker_config) for test in tests: log_prefix = "%s: " % test.name # Build the test image test_docker_file = "%s.dockerfile" % test.name build_log_file = build_log_dir if build_log_dir is not os.devnull: build_log_file = os.path.join( build_log_dir, "%s.log" % test_docker_file.replace(".dockerfile", "").lower()) try: __build(base_url=benchmarker_config.server_docker_host, build_log_file=build_log_file, log_prefix=log_prefix, path=test.directory, dockerfile=test_docker_file, tag="techempower/tfb.test.%s" % test_docker_file.replace(".dockerfile", "")) except Exception: return 1 return 0
def __run_list_test_metadata(self): ''' Prints the metadata for all the available tests ''' all_tests = gather_tests(benchmarker_config=self.config) all_tests_json = json.dumps( map( lambda test: { "name": test.name, "approach": test.approach, "classification": test.classification, "database": test.database, "framework": test.framework, "language": test.language, "orm": test.orm, "platform": test.platform, "webserver": test.webserver, "os": test.os, "database_os": test.database_os, "display_name": test.display_name, "notes": test.notes, "versus": test.versus }, all_tests)) with open(os.path.join(self.results.directory, "test_metadata.json"), "w") as f: f.write(all_tests_json)
def publish(benchmarker_config): ''' Builds fresh versions of all the docker container images known in the system and attempts to publish them to dockerhub. Note: this requires `docker login` to have been performed prior to the call. ''' start_time = time.time() docker_buildargs = { 'CPU_COUNT': str(multiprocessing.cpu_count()), 'MAX_CONCURRENCY': str(max(benchmarker_config.concurrency_levels)), 'TFB_DATABASE': str(benchmarker_config.database_host) } # Force building instead of pulling benchmarker_config.build = True tests = gather_tests(benchmarker_config=benchmarker_config) for test in tests: __build_dependencies(benchmarker_config, test, docker_buildargs) client = docker.DockerClient( base_url=benchmarker_config.server_docker_host) client.login("USERNAME", "PASSWORD") for image in client.images.list(): has_tags = len(image.tags) > 0 if has_tags and 'techempower' in image.tags[ 0] and 'tfb.test' not in image.tags[0]: log("Pushing docker image: %s" % image.tags[0].split(':')[0]) client.images.push(image.tags[0].split(':')[0]) log("Publish took: %s seconds" % (time.time() - start_time))
def build(benchmarker_config, test_names, build_log_dir=os.devnull): ''' Builds the dependency chain as well as the test implementation docker images for the given tests. ''' docker_buildargs = { 'MAX_CONCURRENCY': str(max(benchmarker_config.concurrency_levels)), 'TFB_DATABASE': str(benchmarker_config.database_host) } tests = gather_tests(include=test_names, benchmarker_config=benchmarker_config) for test in tests: log_prefix = "%s: " % test.name if __build_dependencies(benchmarker_config, test, docker_buildargs, build_log_dir) > 0: return 1 # Build the test image test_docker_file = "%s.dockerfile" % test.name build_log_file = build_log_dir if build_log_dir is not os.devnull: build_log_file = os.path.join( build_log_dir, "%s.log" % test_docker_file.replace(".dockerfile", "").lower()) with open(build_log_file, 'w') as build_log: try: for line in docker.APIClient( base_url=benchmarker_config.server_docker_host).build( path=test.directory, dockerfile=test_docker_file, tag="techempower/tfb.test.%s" % test_docker_file.replace(".dockerfile", ""), buildargs=docker_buildargs, forcerm=True): if line.startswith('{"stream":'): line = json.loads(line) line = line[line.keys()[0]].encode('utf-8') log(line, prefix=log_prefix, file=build_log, color=Fore.WHITE + Style.BRIGHT \ if re.match(r'^Step \d+\/\d+', line) else '') except Exception: tb = traceback.format_exc() log("Docker build failed; terminating", prefix=log_prefix, file=build_log, color=Fore.RED) log(tb, prefix=log_prefix, file=build_log) return 1 return 0
def build(benchmarker_config, test_names, build_log_dir=os.devnull): ''' Builds the dependency chain as well as the test implementation docker images for the given tests. ''' tests = gather_tests( include=test_names, benchmarker_config=benchmarker_config) for test in tests: log_prefix = "%s: " % test.name # Build the test image test_docker_file = "%s.dockerfile" % test.name build_log_file = build_log_dir if build_log_dir is not os.devnull: build_log_file = os.path.join( build_log_dir, "%s.log" % test_docker_file.replace(".dockerfile", "").lower()) with open(build_log_file, 'w') as build_log: try: client = docker.APIClient( base_url=benchmarker_config.server_docker_host) output = client.build( path=test.directory, dockerfile=test_docker_file, tag="techempower/tfb.test.%s" % test_docker_file.replace(".dockerfile", ""), forcerm=True, pull=True) buffer = "" for token in output: if token.startswith('{"stream":'): token = json.loads(token) token = token[token.keys()[0]].encode('utf-8') buffer += token while "\n" in buffer: index = buffer.index("\n") line = buffer[:index] buffer = buffer[index + 1:] log(line, prefix=log_prefix, file=build_log, color=Fore.WHITE + Style.BRIGHT \ if re.match(r'^Step \d+\/\d+', line) else '') if buffer: log(buffer, prefix=log_prefix, file=build_log, color=Fore.WHITE + Style.BRIGHT \ if re.match(r'^Step \d+\/\d+', buffer) else '') except Exception: tb = traceback.format_exc() log("Docker build failed; terminating", prefix=log_prefix, file=build_log, color=Fore.RED) log(tb, prefix=log_prefix, file=build_log) return 1 return 0
def main(argv=None): ''' Runs the toolset. ''' # Do argv default this way, as doing it in the functional declaration sets it at compile time if argv is None: argv = sys.argv ########################################################## # Set up argument parser ########################################################## parser = argparse.ArgumentParser( description="Install or run the Framework Benchmarks test suite.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, epilog= '''If an argument includes (type int-sequence), then it accepts integer lists in multiple forms. Using a single number e.g. 5 will create a list [5]. Using commas will create a list containing those values e.g. 1,3,6 creates [1, 3, 6]. Using three colon-separated numbers of start:step:end will create a list, using the semantics of python's range function, e.g. 1:3:15 creates [1, 4, 7, 10, 13] while 0:1:5 creates [0, 1, 2, 3, 4] ''') # Suite options parser.add_argument('--build', nargs='+', help='Builds the dockerfile(s) for the given test(s)') parser.add_argument('--clean', action='store_true', default=False, help='Removes the results directory') parser.add_argument('--new', action='store_true', default=False, help='Initialize a new framework test') parser.add_argument( '--quiet', action='store_true', default=False, help= 'Only print a limited set of messages to stdout, keep the bulk of messages in log files only' ) parser.add_argument( '--results-name', help='Gives a name to this set of results, formatted as a date', default='(unspecified, datetime = %Y-%m-%d %H:%M:%S)') parser.add_argument( '--results-environment', help='Describes the environment in which these results were gathered', default='(unspecified, hostname = %s)' % socket.gethostname()) parser.add_argument( '--results-upload-uri', default=None, help= 'A URI where the in-progress results.json file will be POSTed periodically' ) parser.add_argument( '--parse', help= 'Parses the results of the given timestamp and merges that with the latest results' ) # Test options parser.add_argument('--test', default=None, nargs='+', help='names of tests to run') parser.add_argument( '--test-dir', nargs='+', dest='test_dir', help='name of framework directory containing all tests to run') parser.add_argument( '--test-lang', nargs='+', dest='test_lang', help='name of language directory containing all tests to run') parser.add_argument('--exclude', default=None, nargs='+', help='names of tests to exclude') parser.add_argument('--type', choices=[ 'all', 'json', 'db', 'query', 'cached_query', 'fortune', 'update', 'plaintext' ], default='all', help='which type of test to run') parser.add_argument( '-m', '--mode', choices=['benchmark', 'verify', 'debug'], default='benchmark', help= 'verify mode will only start up the tests, curl the urls and shutdown. debug mode will skip verification and leave the server running.' ) parser.add_argument('--list-tests', action='store_true', default=False, help='lists all the known tests that can run') # Benchmark options parser.add_argument('--duration', default=15, help='Time in seconds that each test should run for.') parser.add_argument('--server-host', default='tfb-server', help='Hostname/IP for application server') parser.add_argument('--database-host', default='tfb-database', help='Hostname/IP for database server') parser.add_argument('--client-host', default='', help='Hostname/IP for client server') parser.add_argument('--concurrency-levels', nargs='+', default=[16, 32, 64, 128, 256, 512], help='List of concurrencies to benchmark') parser.add_argument('--pipeline-concurrency-levels', nargs='+', default=[256, 1024, 4096, 16384], help='List of pipeline concurrencies to benchmark') parser.add_argument('--query-levels', nargs='+', default=[1, 5, 10, 15, 20], help='List of query levels to benchmark') parser.add_argument('--cached-query-levels', nargs='+', default=[1, 10, 20, 50, 100], help='List of cached query levels to benchmark') # Network options parser.add_argument('--network-mode', default=None, help='The network mode to run docker in') args = parser.parse_args() global config config = BenchmarkConfig(args) results = Results(config) if config.new: Scaffolding(config) elif config.build: docker_helper.build(config, config.build) elif config.clean: cleaner.clean(results) docker_helper.clean(config) elif config.list_tests: all_tests = gather_tests(benchmarker_config=config) for test in all_tests: log(test.name) elif config.parse != None: # TODO: broken all_tests = gather_tests(benchmarker_config=config) for test in all_tests: test.parse_all() results.parse(all_tests) else: benchmarker = Benchmarker(config, results) benchmarker.run() return 0