def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', category='all', bq_result_table=None): """Create jobspecs for scenarios to run.""" all_workers = [worker for workers in workers_by_lang.values() for worker in workers] scenarios = [] for language in languages: for scenario_json in language.scenarios(): if re.search(args.regex, scenario_json['name']): if category in scenario_json.get('CATEGORIES', []) or category == 'all': workers = workers_by_lang[str(language)] # 'SERVER_LANGUAGE' is an indicator for this script to pick # a server in different language. custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None) scenario_json = scenario_config.remove_nonproto_fields(scenario_json) if custom_server_lang: if not workers_by_lang.get(custom_server_lang, []): print 'Warning: Skipping scenario %s as' % scenario_json['name'] print('SERVER_LANGUAGE is set to %s yet the language has ' 'not been selected with -l' % custom_server_lang) continue for idx in range(0, scenario_json['num_servers']): # replace first X workers by workers of a different language workers[idx] = workers_by_lang[custom_server_lang][idx] scenario = create_scenario_jobspec(scenario_json, workers, remote_host=remote_host, bq_result_table=bq_result_table) scenarios.append(scenario) # the very last scenario requests shutting down the workers. scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host)) return scenarios
def _mutate_scenario(scenario_json): """Modifies vanilla benchmark scenario config to make it more suitable for running as a unit test.""" # tweak parameters to get fast test times scenario_json = dict(scenario_json) scenario_json['warmup_seconds'] = 0 scenario_json['benchmark_seconds'] = 1 outstanding_rpcs_divisor = 1 if scenario_json['client_config'][ 'client_type'] == 'SYNC_CLIENT' or scenario_json['server_config'][ 'server_type'] == 'SYNC_SERVER': # reduce the number of threads needed for scenarios that use synchronous API outstanding_rpcs_divisor = 10 scenario_json['client_config']['outstanding_rpcs_per_channel'] = max( 1, scenario_json['client_config']['outstanding_rpcs_per_channel'] // outstanding_rpcs_divisor) # Some scenarios use high channel count since when actually # benchmarking, we want to saturate the machine that runs the benchmark. # For unit test, this is an overkill. max_client_channels = 16 if scenario_json['client_config']['rpc_type'] == 'STREAMING_FROM_SERVER': # streaming from server scenarios tend to have trouble shutting down # quickly if there are too many channels. max_client_channels = 4 scenario_json['client_config']['client_channels'] = min( max_client_channels, scenario_json['client_config']['client_channels']) return scenario_config.remove_nonproto_fields(scenario_json)
def _scenario_json_string(scenario_json, is_tsan): scenarios_json = { 'scenarios': [ scenario_config.remove_nonproto_fields( mutate_scenario(scenario_json, is_tsan)) ] } return json.dumps(scenarios_json)
def _scenario_json_string(scenario_json): # tweak parameters to get fast test times scenario_json['warmup_seconds'] = 0 scenario_json['benchmark_seconds'] = 1 scenarios_json = { 'scenarios': [scenario_config.remove_nonproto_fields(scenario_json)] } return json.dumps(scenarios_json)
def _mutate_scenario(scenario_json): """Modifies vanilla benchmark scenario config to make it more suitable for running as a unit test.""" # tweak parameters to get fast test times scenario_json = dict(scenario_json) scenario_json['warmup_seconds'] = 0 scenario_json['benchmark_seconds'] = 1 outstanding_rpcs_divisor = 1 if scenario_json['client_config'][ 'client_type'] == 'SYNC_CLIENT' or scenario_json['server_config'][ 'server_type'] == 'SYNC_SERVER': # reduce the number of threads needed for scenarios that use synchronous API outstanding_rpcs_divisor = 10 scenario_json['client_config']['outstanding_rpcs_per_channel'] = max( 1, scenario_json['client_config']['outstanding_rpcs_per_channel'] // outstanding_rpcs_divisor) return scenario_config.remove_nonproto_fields(scenario_json)
def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', category='all', bq_result_table=None): """Create jobspecs for scenarios to run.""" all_workers = [ worker for workers in workers_by_lang.values() for worker in workers ] scenarios = [] for language in languages: for scenario_json in language.scenarios(): if re.search(args.regex, scenario_json['name']): if category in scenario_json.get('CATEGORIES', []) or category == 'all': workers = workers_by_lang[str(language)] # 'SERVER_LANGUAGE' is an indicator for this script to pick # a server in different language. custom_server_lang = scenario_json.get( 'SERVER_LANGUAGE', None) scenario_json = scenario_config.remove_nonproto_fields( scenario_json) if custom_server_lang: if not workers_by_lang.get(custom_server_lang, []): print 'Warning: Skipping scenario %s as' % scenario_json[ 'name'] print( 'SERVER_LANGUAGE is set to %s yet the language has ' 'not been selected with -l' % custom_server_lang) continue for idx in range(0, scenario_json['num_servers']): # replace first X workers by workers of a different language workers[idx] = workers_by_lang[custom_server_lang][ idx] scenario = create_scenario_jobspec( scenario_json, workers, remote_host=remote_host, bq_result_table=bq_result_table) scenarios.append(scenario) # the very last scenario requests shutting down the workers. scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host)) return scenarios
def _scenario_json_string(scenario_json): # tweak parameters to get fast test times scenario_json['warmup_seconds'] = 1 scenario_json['benchmark_seconds'] = 1 return json.dumps(scenario_config.remove_nonproto_fields(scenario_json))
def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', category='all', bq_result_table=None, netperf=False, netperf_hosts=[], server_cpu_load=0): """Create jobspecs for scenarios to run.""" all_workers = [ worker for workers in workers_by_lang.values() for worker in workers ] scenarios = [] _NO_WORKERS = [] if netperf: if not netperf_hosts: netperf_server = 'localhost' netperf_client = None elif len(netperf_hosts) == 1: netperf_server = netperf_hosts[0] netperf_client = netperf_hosts[0] else: netperf_server = netperf_hosts[0] netperf_client = netperf_hosts[1] scenarios.append( Scenario( create_netperf_jobspec(server_host=netperf_server, client_host=netperf_client, bq_result_table=bq_result_table), _NO_WORKERS, 'netperf')) for language in languages: for scenario_json in language.scenarios(): if re.search(args.regex, scenario_json['name']): categories = scenario_json.get('CATEGORIES', ['scalable', 'smoketest']) if category in categories or category == 'all': workers = workers_by_lang[str(language)][:] # 'SERVER_LANGUAGE' is an indicator for this script to pick # a server in different language. custom_server_lang = scenario_json.get( 'SERVER_LANGUAGE', None) custom_client_lang = scenario_json.get( 'CLIENT_LANGUAGE', None) scenario_json = scenario_config.remove_nonproto_fields( scenario_json) if custom_server_lang and custom_client_lang: raise Exception( 'Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE' 'in the same scenario') if custom_server_lang: if not workers_by_lang.get(custom_server_lang, []): print('Warning: Skipping scenario %s as' % scenario_json['name']) print( 'SERVER_LANGUAGE is set to %s yet the language has ' 'not been selected with -l' % custom_server_lang) continue for idx in range(0, scenario_json['num_servers']): # replace first X workers by workers of a different language workers[idx] = workers_by_lang[custom_server_lang][ idx] if custom_client_lang: if not workers_by_lang.get(custom_client_lang, []): print('Warning: Skipping scenario %s as' % scenario_json['name']) print( 'CLIENT_LANGUAGE is set to %s yet the language has ' 'not been selected with -l' % custom_client_lang) continue for idx in range(scenario_json['num_servers'], len(workers)): # replace all client workers by workers of a different language, # leave num_server workers as they are server workers. workers[idx] = workers_by_lang[custom_client_lang][ idx] scenario = Scenario( create_scenario_jobspec( scenario_json, [w.host_and_port for w in workers], remote_host=remote_host, bq_result_table=bq_result_table, server_cpu_load=server_cpu_load), workers, scenario_json['name']) scenarios.append(scenario) return scenarios
def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', category='all', bq_result_table=None, netperf=False, netperf_hosts=[], server_cpu_load=0): """Create jobspecs for scenarios to run.""" all_workers = [worker for workers in workers_by_lang.values() for worker in workers] scenarios = [] _NO_WORKERS = [] if netperf: if not netperf_hosts: netperf_server='localhost' netperf_client=None elif len(netperf_hosts) == 1: netperf_server=netperf_hosts[0] netperf_client=netperf_hosts[0] else: netperf_server=netperf_hosts[0] netperf_client=netperf_hosts[1] scenarios.append(Scenario( create_netperf_jobspec(server_host=netperf_server, client_host=netperf_client, bq_result_table=bq_result_table), _NO_WORKERS, 'netperf')) for language in languages: for scenario_json in language.scenarios(): if re.search(args.regex, scenario_json['name']): categories = scenario_json.get('CATEGORIES', ['scalable', 'smoketest']) if category in categories or category == 'all': workers = workers_by_lang[str(language)][:] # 'SERVER_LANGUAGE' is an indicator for this script to pick # a server in different language. custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None) custom_client_lang = scenario_json.get('CLIENT_LANGUAGE', None) scenario_json = scenario_config.remove_nonproto_fields(scenario_json) if custom_server_lang and custom_client_lang: raise Exception('Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE' 'in the same scenario') if custom_server_lang: if not workers_by_lang.get(custom_server_lang, []): print('Warning: Skipping scenario %s as' % scenario_json['name']) print('SERVER_LANGUAGE is set to %s yet the language has ' 'not been selected with -l' % custom_server_lang) continue for idx in range(0, scenario_json['num_servers']): # replace first X workers by workers of a different language workers[idx] = workers_by_lang[custom_server_lang][idx] if custom_client_lang: if not workers_by_lang.get(custom_client_lang, []): print('Warning: Skipping scenario %s as' % scenario_json['name']) print('CLIENT_LANGUAGE is set to %s yet the language has ' 'not been selected with -l' % custom_client_lang) continue for idx in range(scenario_json['num_servers'], len(workers)): # replace all client workers by workers of a different language, # leave num_server workers as they are server workers. workers[idx] = workers_by_lang[custom_client_lang][idx] scenario = Scenario( create_scenario_jobspec(scenario_json, [w.host_and_port for w in workers], remote_host=remote_host, bq_result_table=bq_result_table, server_cpu_load=server_cpu_load), workers, scenario_json['name']) scenarios.append(scenario) return scenarios
def _scenario_json_string(scenario_json, is_tsan): scenarios_json = {'scenarios': [scenario_config.remove_nonproto_fields(mutate_scenario(scenario_json, is_tsan))]} return json.dumps(scenarios_json)
def _scenario_json_string(scenario_json): return json.dumps(scenario_config.remove_nonproto_fields(scenario_json))
def create_scenarios( languages, workers_by_lang, remote_host=None, regex=".*", category="all", bq_result_table=None, netperf=False, netperf_hosts=[], ): """Create jobspecs for scenarios to run.""" all_workers = [worker for workers in workers_by_lang.values() for worker in workers] scenarios = [] if netperf: if not netperf_hosts: netperf_server = "localhost" netperf_client = None elif len(netperf_hosts) == 1: netperf_server = netperf_hosts[0] netperf_client = netperf_hosts[0] else: netperf_server = netperf_hosts[0] netperf_client = netperf_hosts[1] scenarios.append( create_netperf_jobspec( server_host=netperf_server, client_host=netperf_client, bq_result_table=bq_result_table ) ) for language in languages: for scenario_json in language.scenarios(): if re.search(args.regex, scenario_json["name"]): categories = scenario_json.get("CATEGORIES", []) if category in categories or (category == "all" and categories != ["sweep"]): workers = workers_by_lang[str(language)] # 'SERVER_LANGUAGE' is an indicator for this script to pick # a server in different language. custom_server_lang = scenario_json.get("SERVER_LANGUAGE", None) custom_client_lang = scenario_json.get("CLIENT_LANGUAGE", None) scenario_json = scenario_config.remove_nonproto_fields(scenario_json) if custom_server_lang and custom_client_lang: raise Exception( "Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE" "in the same scenario" ) if custom_server_lang: if not workers_by_lang.get(custom_server_lang, []): print("Warning: Skipping scenario %s as" % scenario_json["name"]) print( "SERVER_LANGUAGE is set to %s yet the language has " "not been selected with -l" % custom_server_lang ) continue for idx in range(0, scenario_json["num_servers"]): # replace first X workers by workers of a different language workers[idx] = workers_by_lang[custom_server_lang][idx] if custom_client_lang: if not workers_by_lang.get(custom_client_lang, []): print("Warning: Skipping scenario %s as" % scenario_json["name"]) print( "CLIENT_LANGUAGE is set to %s yet the language has " "not been selected with -l" % custom_client_lang ) continue for idx in range(scenario_json["num_servers"], len(workers)): # replace all client workers by workers of a different language, # leave num_server workers as they are server workers. workers[idx] = workers_by_lang[custom_client_lang][idx] scenario = create_scenario_jobspec( scenario_json, workers, remote_host=remote_host, bq_result_table=bq_result_table ) scenarios.append(scenario) # the very last scenario requests shutting down the workers. scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host)) return scenarios