def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, flake_retries=0, timeout_retries=0): """Creates jobspec for a task running under docker.""" environ = environ.copy() environ['RUN_COMMAND'] = shell_command docker_args=[] for k,v in environ.iteritems(): docker_args += ['-e', '%s=%s' % (k, v)] docker_env = {'DOCKERFILE_DIR': dockerfile_dir, 'DOCKER_RUN_SCRIPT': 'tools/jenkins/docker_run.sh', 'OUTPUT_DIR': 'artifacts'} jobspec = jobset.JobSpec( cmdline=['tools/jenkins/build_and_run_docker.sh'] + docker_args, environ=docker_env, shortname='build_artifact.%s' % (name), timeout_seconds=30*60, flake_retries=flake_retries, timeout_retries=timeout_retries) return jobspec
def server_jobspec(language, docker_image): """Create jobspec for running a server""" container_name = dockerjob.random_name('interop_server_%s' % language.safename) cmdline = bash_login_cmdline( language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT])) environ = language.global_env() docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, cwd=language.server_cwd, environ=environ, docker_args=['-p', str(_DEFAULT_SERVER_PORT), '--name', container_name]) server_job = jobset.JobSpec( cmdline=docker_cmdline, environ=environ, shortname='interop_server_%s' % language, timeout_seconds=30*60) server_job.container_name = container_name return server_job
def create_scenario_jobspec(scenario_json, workers, remote_host=None, bq_result_table=None): """Runs one scenario using QPS driver.""" # setting QPS_WORKERS env variable here makes sure it works with SSH too. cmd = 'QPS_WORKERS="%s" ' % ','.join(workers) if bq_result_table: cmd += 'BQ_RESULT_TABLE="%s" ' % bq_result_table cmd += 'tools/run_tests/performance/run_qps_driver.sh ' cmd += '--scenarios_json=%s ' % pipes.quote(json.dumps({'scenarios': [scenario_json]})) cmd += '--scenario_result_file=scenario_result.json' if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd)) return jobset.JobSpec( cmdline=[cmd], shortname='qps_json_driver.%s' % scenario_json['name'], timeout_seconds=3*60, shell=True, verbose_success=True)
def create_qpsworker_job(language, shortname=None, port=10000, remote_host=None): # TODO: support more languages cmdline = language.worker_cmdline() + ['--driver_port=%s' % port] if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) cmdline = ['ssh', str(user_at_host), 'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline)] host_and_port='%s:%s' % (remote_host, port) else: host_and_port='localhost:%s' % port # TODO(jtattermusch): with some care, we can calculate the right timeout # of a worker from the sum of warmup + benchmark times for all the scenarios jobspec = jobset.JobSpec( cmdline=cmdline, shortname=shortname, timeout_seconds=15*60) return QpsWorkerJob(jobspec, language, host_and_port)
def job_spec(self, cmdline, hash_targets, shortname=None, environ={}): """Construct a jobset.JobSpec for a test under this config Args: cmdline: a list of strings specifying the command line the test would like to run hash_targets: either None (don't do caching of test results), or a list of strings specifying files to include in a binary hash to check if a test has changed -- if used, all artifacts needed to run the test must be listed """ actual_environ = self.environ.copy() for k, v in environ.iteritems(): actual_environ[k] = v return jobset.JobSpec( cmdline=cmdline, shortname=shortname, environ=actual_environ, hash_targets=hash_targets if self.allow_hashing else None)
def cloud_to_cloud_jobspec(language, test_cases, server_addresses, test_duration_secs, num_channels_per_server, num_stubs_per_channel, metrics_port, docker_image=None): """Creates jobspec for cloud-to-cloud interop test""" cmdline = bash_login_cmdline(language.client_cmd([ '--test_cases=%s' % test_cases, '--server_addresses=%s' % server_addresses, '--test_duration_secs=%s' % test_duration_secs, '--num_stubs_per_channel=%s' % num_stubs_per_channel, '--num_channels_per_server=%s' % num_channels_per_server, '--metrics_port=%s' % metrics_port ])) print cmdline cwd = language.client_cwd environ = language.global_env() if docker_image: container_name = dockerjob.random_name('interop_client_%s' % language.safename) cmdline = docker_run_cmdline( cmdline, image=docker_image, environ=environ, cwd=cwd, docker_args=['--net=host', '--name', container_name]) cwd = None test_job = jobset.JobSpec(cmdline=cmdline, cwd=cwd, environ=environ, shortname='cloud_to_cloud:%s:%s_server:stress_test' % ( language, server_name), timeout_seconds=test_duration_secs * 2, flake_retries=0, timeout_retries=0, kill_handler=_job_kill_handler) test_job.container_name = container_name return test_job
def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False): """Creates jobspec for cloud-to-prod interop test""" container_name = None cmdargs = [ '--server_host_override=grpc-test.sandbox.google.com', '--server_host=grpc-test.sandbox.google.com', '--server_port=443', '--use_tls=true', '--test_case=%s' % test_case ] environ = dict(language.cloud_to_prod_env(), **language.global_env()) if auth: auth_cmdargs, auth_env = auth_options(language, test_case) cmdargs += auth_cmdargs environ.update(auth_env) cmdline = bash_login_cmdline(language.client_cmd(cmdargs)) cwd = language.client_cwd if docker_image: container_name = dockerjob.random_name('interop_client_%s' % language.safename) cmdline = docker_run_cmdline( cmdline, image=docker_image, cwd=cwd, environ=environ, docker_args=['--net=host', '--name', container_name]) cwd = None environ = None suite_name = 'cloud_to_prod_auth' if auth else 'cloud_to_prod' test_job = jobset.JobSpec(cmdline=cmdline, cwd=cwd, environ=environ, shortname='%s:%s:%s' % (suite_name, language, test_case), timeout_seconds=90, flake_retries=5 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, kill_handler=_job_kill_handler) test_job.container_name = container_name return test_job
def _generate_test_case_jobspecs(lang, runtime, release, suite_name): """Returns the list of test cases from testcase files per lang/release.""" testcase_lines = _read_test_cases_file(lang, runtime, release) job_spec_list = [] for line in testcase_lines: # TODO(jtattermusch): revisit the logic for updating test case commands # what it currently being done seems fragile. # Extract test case name from the command line m = re.search(r'--test_case=(\w+)', line) testcase_name = m.group(1) if m else 'unknown_test' # Extract the server name from the command line if '--server_host_override=' in line: m = re.search( r'--server_host_override=((.*).sandbox.googleapis.com)', line) else: m = re.search(r'--server_host=((.*).sandbox.googleapis.com)', line) server = m.group(1) if m else 'unknown_server' server_short = m.group(2) if m else 'unknown_server' # replace original server_host argument assert '--server_host=' in line line = re.sub(r'--server_host=[^ ]*', r'--server_host=%s' % args.server_host, line) # some interop tests don't set server_host_override (see #17407), # but we need to use it if different host is set via cmdline args. if args.server_host != server and not '--server_host_override=' in line: line = re.sub(r'(--server_host=[^ ]*)', r'\1 --server_host_override=%s' % server, line) spec = jobset.JobSpec(cmdline=line, shortname='%s:%s:%s:%s' % (suite_name, lang, server_short, testcase_name), timeout_seconds=_TEST_TIMEOUT_SECONDS, shell=True, flake_retries=5 if args.allow_flakes else 0) job_spec_list.append(spec) return job_spec_list
def archive_repo(): """Archives local version of repo including submodules.""" # TODO: also archive grpc-go and grpc-java repos archive_job = jobset.JobSpec( cmdline=['tar', '-cf', '../grpc.tar', '../grpc/'], shortname='archive_repo', timeout_seconds=3 * 60) jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run([archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository create successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to archive local repository.', do_newline=True) sys.exit(1)
def build_interop_image_jobspec(language, tag=None): """Creates jobspec for building interop docker image for a language""" if not tag: tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4()) env = {'INTEROP_IMAGE': tag, 'BASE_NAME': 'grpc_interop_%s' % language.safename} if not args.travis: env['TTY_FLAG'] = '-t' # This env variable is used to get around the github rate limit # error when running the PHP `composer install` command # TODO(stanleycheung): find a more elegant way to do this if language.safename == 'php' and os.path.exists('/var/local/.composer/auth.json'): env['BUILD_INTEROP_DOCKER_EXTRA_ARGS'] = \ '-v /var/local/.composer/auth.json:/root/.composer/auth.json:ro' build_job = jobset.JobSpec( cmdline=['tools/jenkins/build_interop_image.sh'], environ=env, shortname='build_docker_%s' % (language), timeout_seconds=30*60) build_job.tag = tag return build_job
def create_qpsworker_job(language, shortname=None, port=10000, remote_host=None): # TODO: support more languages cmdline = language.worker_cmdline() + ['--driver_port=%s' % port] if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) cmdline = [ 'ssh', str(user_at_host), 'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline) ] host_and_port = '%s:%s' % (remote_host, port) else: host_and_port = 'localhost:%s' % port jobspec = jobset.JobSpec(cmdline=cmdline, shortname=shortname, timeout_seconds=15 * 60) return QpsWorkerJob(jobspec, language, host_and_port)
def build_image_jobspec(runtime, env, gcr_tag, stack_base): """Build interop docker image for a language with runtime. runtime: a <lang><version> string, for example go1.8. env: dictionary of env to passed to the build script. gcr_tag: the tag for the docker image (i.e. v1.3.0). stack_base: the local gRPC repo path. """ basename = 'grpc_interop_%s' % runtime tag = '%s/%s:%s' % (args.gcr_path, basename, gcr_tag) build_env = {'INTEROP_IMAGE': tag, 'BASE_NAME': basename, 'TTY_FLAG': '-t'} build_env.update(env) image_builder_path = _IMAGE_BUILDER if client_matrix.should_build_docker_interop_image_from_release_tag(lang): image_builder_path = os.path.join(stack_base, _IMAGE_BUILDER) build_job = jobset.JobSpec(cmdline=[image_builder_path], environ=build_env, shortname='build_docker_%s' % runtime, timeout_seconds=30 * 60) build_job.tag = tag return build_job
def _collect_bm_data(bm, cfg, name, reps, idx, loops): jobs_list = [] for line in subprocess.check_output( ['bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_list_tests']).splitlines(): stripped_line = line.strip().replace("/", "_").replace( "<", "_").replace(">", "_").replace(", ", "_") cmd = [ 'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_filter=^%s$' % line, '--benchmark_out=%s.%s.%s.%s.%d.json' % (bm, stripped_line, cfg, name, idx), '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (reps) ] jobs_list.append( jobset.JobSpec( cmd, shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1, loops), verbose_success=True, timeout_seconds=60 * 2)) return jobs_list
def build_interop_image_jobspec(language, tag=None): """Creates jobspec for building interop docker image for a language""" if not tag: tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4()) env = {'INTEROP_IMAGE': tag, 'BASE_NAME': 'grpc_interop_%s' % language.safename} if not args.travis: env['TTY_FLAG'] = '-t' # This env variable is used to get around the github rate limit # error when running the PHP `composer install` command host_file = '%s/.composer/auth.json' % os.environ['HOME'] if language.safename == 'php' and os.path.exists(host_file): env['BUILD_INTEROP_DOCKER_EXTRA_ARGS'] = \ '-v %s:/root/.composer/auth.json:ro' % host_file build_job = jobset.JobSpec( cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], environ=env, shortname='build_docker_%s' % (language), timeout_seconds=30*60) build_job.tag = tag return build_job
def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, server_port, docker_image=None): """Creates jobspec for cloud-to-cloud interop test""" cmdline = bash_login_cmdline( language.client_cmd([ '--server_host_override=foo.test.google.fr', '--use_tls=true', '--use_test_ca=true', '--test_case=%s' % test_case, '--server_host=%s' % server_host, '--server_port=%s' % server_port ])) cwd = language.client_cwd environ = language.global_env() if docker_image: container_name = dockerjob.random_name('interop_client_%s' % language.safename) cmdline = docker_run_cmdline( cmdline, image=docker_image, environ=environ, cwd=cwd, docker_args=['--net=host', '--name', container_name]) cwd = None test_job = jobset.JobSpec(cmdline=cmdline, cwd=cwd, environ=environ, shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name, test_case), timeout_seconds=_TEST_TIMEOUT, flake_retries=5 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, kill_handler=_job_kill_handler) if docker_image: test_job.container_name = container_name return test_job
def create_qpsworker_job(language, shortname=None, port=10000, remote_host=None): cmdline = language.worker_cmdline() + ['--driver_port=%s' % port] if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) cmdline = [ 'ssh', str(user_at_host), 'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline) ] host_and_port = '%s:%s' % (remote_host, port) else: host_and_port = 'localhost:%s' % port jobspec = jobset.JobSpec( cmdline=cmdline, shortname=shortname, timeout_seconds=5 * 60, # workers get restarted after each scenario verbose_success=True) return QpsWorkerJob(jobspec, language, host_and_port)
def prepare_remote_hosts(hosts): """Prepares remote hosts.""" prepare_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) prepare_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/remote_host_prepare.sh'], shortname='remote_host_prepare.%s' % host, environ = {'USER_AT_HOST': user_at_host}, timeout_seconds=5*60)) jobset.message('START', 'Preparing remote hosts.', do_newline=True) num_failures, _ = jobset.run( prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Remote hosts ready to start build.', do_newline=True) else: jobset.message('FAILED', 'Failed to prepare remote hosts.', do_newline=True) sys.exit(1)
def find_test_cases(lang, runtime, release, suite_name): """Returns the list of test cases from testcase files per lang/release.""" file_tmpl = os.path.join(os.path.dirname(__file__), 'testcases/%s__%s') testcase_release = release filename_prefix = lang if lang == 'csharp': filename_prefix = runtime if not os.path.exists(file_tmpl % (filename_prefix, release)): testcase_release = 'master' testcases = file_tmpl % (filename_prefix, testcase_release) job_spec_list = [] try: with open(testcases) as f: # Only line start with 'docker run' are test cases. for line in f.readlines(): if line.startswith('docker run'): m = re.search('--test_case=(.*)"', line) shortname = m.group(1) if m else 'unknown_test' m = re.search( '--server_host_override=(.*).sandbox.googleapis.com', line) server = m.group(1) if m else 'unknown_server' spec = jobset.JobSpec( cmdline=line, shortname='%s:%s:%s:%s' % (suite_name, lang, server, shortname), timeout_seconds=_TEST_TIMEOUT, shell=True, flake_retries=5 if args.allow_flakes else 0) job_spec_list.append(spec) jobset.message('START', 'Loaded %s tests from %s' % (len(job_spec_list), testcases), do_newline=True) except IOError as err: jobset.message('FAILED', err, do_newline=True) return job_spec_list
make_targets = {} for l in languages: makefile = l.makefile_name() make_targets[makefile] = make_targets.get(makefile, set()).union( set(l.make_targets())) def build_step_environ(cfg): environ = {'CONFIG': cfg} msbuild_cfg = _MSBUILD_CONFIG.get(cfg) if msbuild_cfg: environ['MSBUILD_CONFIG'] = msbuild_cfg return environ build_steps = list(set( jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5) for l in languages for cmdline in l.pre_build_steps())) if make_targets: make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems()) build_steps.extend(set(make_commands)) build_steps.extend(set( jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None) for l in languages for cmdline in l.build_steps())) post_tests_steps = list(set( jobset.JobSpec(cmdline, environ=build_step_environ(build_config)) for l in languages for cmdline in l.post_tests_steps())) runs_per_test = args.runs_per_test
def job_spec(self, binary, hash_targets): return jobset.JobSpec( cmdline=['valgrind', '--tool=%s' % self.tool] + self.args + [binary], shortname='valgrind %s' % binary, hash_targets=None)
base_cmd = ['python2.7', 'tools/buildgen/mako_renderer.py'] cmd = base_cmd[:] for plugin in plugins: cmd.append('-p') cmd.append(plugin) for js in json: cmd.append('-d') cmd.append(js) cmd.append('-w') preprocessed_build = '.preprocessed_build' cmd.append(preprocessed_build) if args.output_merged is not None: cmd.append('-M') cmd.append(args.output_merged) pre_jobs.append( jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None)) jobs = [] for template in reversed(sorted(templates)): root, f = os.path.split(template) if os.path.splitext(f)[1] == '.template': out_dir = '.' + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) cmd = base_cmd[:] cmd.append('-P') cmd.append(preprocessed_build) cmd.append('-o') if test is None: cmd.append(out)
nargs='+', default=sorted(_LANGUAGES.keys())) args = argp.parse_args() # grab config run_configs = set(_CONFIGS[cfg] for cfg in itertools.chain.from_iterable( _CONFIGS.iterkeys() if x == 'all' else [x] for x in args.config)) build_configs = set(cfg.build_config for cfg in run_configs) make_targets = [] languages = set(_LANGUAGES[l] for l in args.language) build_steps = [ jobset.JobSpec([ 'make', '-j', '%d' % (multiprocessing.cpu_count() + 1), 'CONFIG=%s' % cfg ] + list( set(itertools.chain.from_iterable(l.make_targets() for l in languages)))) for cfg in build_configs ] + list( set( jobset.JobSpec(cmdline) for l in languages for cmdline in l.build_steps())) one_run = set(spec for config in run_configs for language in args.language for spec in _LANGUAGES[language].test_specs(config) if re.search(args.regex, spec.shortname)) runs_per_test = args.runs_per_test forever = args.forever
def make_jobspec(cfg, targets): return jobset.JobSpec(['make.bat', 'CONFIG=%s' % cfg] + targets, cwd='vsprojects', shell=True)
build_configs = set(cfg.build_config for cfg in run_configs) make_targets = [] languages = set(_LANGUAGES[l] for l in args.language) if len(build_configs) > 1: for language in languages: if not language.supports_multi_config(): print language, 'does not support multiple build configurations' sys.exit(1) build_steps = [ jobset.JobSpec([ 'make', '-j', '%d' % (multiprocessing.cpu_count() + 1), 'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % args.slowdown, 'CONFIG=%s' % cfg ] + list( set(itertools.chain.from_iterable(l.make_targets() for l in languages)))) for cfg in build_configs ] + list( set( jobset.JobSpec(cmdline, environ={'CONFIG': cfg}) for cfg in build_configs for l in languages for cmdline in l.build_steps())) one_run = set(spec for config in run_configs for language in args.language for spec in _LANGUAGES[language].test_specs(config, args.travis) if re.search(args.regex, spec.shortname)) runs_per_test = args.runs_per_test forever = args.forever
return jobset.JobSpec(['make.bat', 'CONFIG=%s' % cfg] + targets, cwd='vsprojects', shell=True) else: def make_jobspec(cfg, targets): return jobset.JobSpec(['make', '-j', '%d' % (multiprocessing.cpu_count() + 1), 'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % args.slowdown, 'CONFIG=%s' % cfg] + targets) build_steps = [make_jobspec(cfg, list(set(itertools.chain.from_iterable( l.make_targets() for l in languages)))) for cfg in build_configs] build_steps.extend(set( jobset.JobSpec(cmdline, environ={'CONFIG': cfg}) for cfg in build_configs for l in languages for cmdline in l.build_steps())) one_run = set( spec for config in run_configs for language in languages for spec in language.test_specs(config, args.travis) if re.search(args.regex, spec.shortname)) runs_per_test = args.runs_per_test forever = args.forever class TestCache(object):
args.slowdown, 'CONFIG=%s' % cfg ] + targets, timeout_seconds=30 * 60) ] make_targets = {} for l in languages: makefile = l.makefile_name() make_targets[makefile] = make_targets.get(makefile, set()).union( set(l.make_targets())) build_steps = list( set( jobset.JobSpec(cmdline, environ={'CONFIG': cfg}, flake_retries=5) for cfg in build_configs for l in languages for cmdline in l.pre_build_steps())) if make_targets: make_commands = itertools.chain.from_iterable( make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems()) build_steps.extend(set(make_commands)) build_steps.extend( set( jobset.JobSpec( cmdline, environ={'CONFIG': cfg}, timeout_seconds=10 * 60) for cfg in build_configs for l in languages for cmdline in l.build_steps())) runs_per_test = args.runs_per_test
def job_spec(self, cmdline, hash_targets): return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] + self.args + cmdline, shortname='valgrind %s' % cmdline[0], hash_targets=None)
def job_spec(self, binary, hash_targets): return jobset.JobSpec( cmdline=[binary], environ=self.environ, hash_targets=hash_targets if self.allow_hashing else None)
def make_jobspec(cfg, targets): return jobset.JobSpec(['make', '-j', '%d' % (multiprocessing.cpu_count() + 1), 'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % args.slowdown, 'CONFIG=%s' % cfg] + targets)
'--config=' + config, ] if args.fix: cmdline.append('--fix') if args.only_changed: orig_files = set(args.files) actual_files = [] output = subprocess.check_output( ['git', 'diff', 'origin/master', 'HEAD', '--name-only']) for line in output.decode('ascii').splitlines(False): if line in orig_files: print("check: %s" % line) actual_files.append(line) else: print("skip: %s - not in the build" % line) args.files = actual_files jobs = [] for filename in args.files: jobs.append( jobset.JobSpec( cmdline + [filename], shortname=filename, timeout_seconds=15 * 60, )) num_fails, res_set = jobset.run(jobs, maxjobs=args.jobs, quiet_success=True) sys.exit(num_fails)