def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
category='all', bq_result_table=None):
"""Create jobspecs for scenarios to run."""
all_workers = [worker
for workers in workers_by_lang.values()
for worker in workers]
scenarios = []
for language in languages:
for scenario_json in language.scenarios():
if re.search(args.regex, scenario_json['name']):
if category in scenario_json.get('CATEGORIES', []) or category == 'all':
workers = workers_by_lang[str(language)]
# 'SERVER_LANGUAGE' is an indicator for this script to pick
# a server in different language.
custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None)
scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
print 'Warning: Skipping scenario %s as' % scenario_json['name']
print('SERVER_LANGUAGE is set to %s yet the language has '
'not been selected with -l' % custom_server_lang)
continue
for idx in range(0, scenario_json['num_servers']):
# replace first X workers by workers of a different language
workers[idx] = workers_by_lang[custom_server_lang][idx]
scenario = create_scenario_jobspec(scenario_json,
workers,
remote_host=remote_host,
bq_result_table=bq_result_table)
scenarios.append(scenario)
# the very last scenario requests shutting down the workers.
scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host))
return scenarios
def _mutate_scenario(scenario_json):
"""Modifies vanilla benchmark scenario config to make it more suitable for running as a unit test."""
# tweak parameters to get fast test times
scenario_json = dict(scenario_json)
scenario_json['warmup_seconds'] = 0
scenario_json['benchmark_seconds'] = 1
outstanding_rpcs_divisor = 1
if scenario_json['client_config'][
'client_type'] == 'SYNC_CLIENT' or scenario_json['server_config'][
'server_type'] == 'SYNC_SERVER':
# reduce the number of threads needed for scenarios that use synchronous API
outstanding_rpcs_divisor = 10
scenario_json['client_config']['outstanding_rpcs_per_channel'] = max(
1, scenario_json['client_config']['outstanding_rpcs_per_channel'] //
outstanding_rpcs_divisor)
# Some scenarios use high channel count since when actually
# benchmarking, we want to saturate the machine that runs the benchmark.
# For unit test, this is an overkill.
max_client_channels = 16
if scenario_json['client_config']['rpc_type'] == 'STREAMING_FROM_SERVER':
# streaming from server scenarios tend to have trouble shutting down
# quickly if there are too many channels.
max_client_channels = 4
scenario_json['client_config']['client_channels'] = min(
max_client_channels, scenario_json['client_config']['client_channels'])
return scenario_config.remove_nonproto_fields(scenario_json)
def _scenario_json_string(scenario_json, is_tsan):
scenarios_json = {
'scenarios': [
scenario_config.remove_nonproto_fields(
mutate_scenario(scenario_json, is_tsan))
]
}
return json.dumps(scenarios_json)
def _scenario_json_string(scenario_json):
# tweak parameters to get fast test times
scenario_json['warmup_seconds'] = 0
scenario_json['benchmark_seconds'] = 1
scenarios_json = {
'scenarios': [scenario_config.remove_nonproto_fields(scenario_json)]
}
return json.dumps(scenarios_json)
def _mutate_scenario(scenario_json):
"""Modifies vanilla benchmark scenario config to make it more suitable for running as a unit test."""
# tweak parameters to get fast test times
scenario_json = dict(scenario_json)
scenario_json['warmup_seconds'] = 0
scenario_json['benchmark_seconds'] = 1
outstanding_rpcs_divisor = 1
if scenario_json['client_config'][
'client_type'] == 'SYNC_CLIENT' or scenario_json['server_config'][
'server_type'] == 'SYNC_SERVER':
# reduce the number of threads needed for scenarios that use synchronous API
outstanding_rpcs_divisor = 10
scenario_json['client_config']['outstanding_rpcs_per_channel'] = max(
1, scenario_json['client_config']['outstanding_rpcs_per_channel'] //
outstanding_rpcs_divisor)
return scenario_config.remove_nonproto_fields(scenario_json)
def create_scenarios(languages,
workers_by_lang,
remote_host=None,
regex='.*',
category='all',
bq_result_table=None):
"""Create jobspecs for scenarios to run."""
all_workers = [
worker for workers in workers_by_lang.values() for worker in workers
]
scenarios = []
for language in languages:
for scenario_json in language.scenarios():
if re.search(args.regex, scenario_json['name']):
if category in scenario_json.get('CATEGORIES',
[]) or category == 'all':
workers = workers_by_lang[str(language)]
# 'SERVER_LANGUAGE' is an indicator for this script to pick
# a server in different language.
custom_server_lang = scenario_json.get(
'SERVER_LANGUAGE', None)
scenario_json = scenario_config.remove_nonproto_fields(
scenario_json)
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
print 'Warning: Skipping scenario %s as' % scenario_json[
'name']
print(
'SERVER_LANGUAGE is set to %s yet the language has '
'not been selected with -l' %
custom_server_lang)
continue
for idx in range(0, scenario_json['num_servers']):
# replace first X workers by workers of a different language
workers[idx] = workers_by_lang[custom_server_lang][
idx]
scenario = create_scenario_jobspec(
scenario_json,
workers,
remote_host=remote_host,
bq_result_table=bq_result_table)
scenarios.append(scenario)
# the very last scenario requests shutting down the workers.
scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host))
return scenarios
def _scenario_json_string(scenario_json): # tweak parameters to get fast test times scenario_json['warmup_seconds'] = 1 scenario_json['benchmark_seconds'] = 1 return json.dumps(scenario_config.remove_nonproto_fields(scenario_json))
def create_scenarios(languages,
workers_by_lang,
remote_host=None,
regex='.*',
category='all',
bq_result_table=None,
netperf=False,
netperf_hosts=[],
server_cpu_load=0):
"""Create jobspecs for scenarios to run."""
all_workers = [
worker for workers in workers_by_lang.values() for worker in workers
]
scenarios = []
_NO_WORKERS = []
if netperf:
if not netperf_hosts:
netperf_server = 'localhost'
netperf_client = None
elif len(netperf_hosts) == 1:
netperf_server = netperf_hosts[0]
netperf_client = netperf_hosts[0]
else:
netperf_server = netperf_hosts[0]
netperf_client = netperf_hosts[1]
scenarios.append(
Scenario(
create_netperf_jobspec(server_host=netperf_server,
client_host=netperf_client,
bq_result_table=bq_result_table),
_NO_WORKERS, 'netperf'))
for language in languages:
for scenario_json in language.scenarios():
if re.search(args.regex, scenario_json['name']):
categories = scenario_json.get('CATEGORIES',
['scalable', 'smoketest'])
if category in categories or category == 'all':
workers = workers_by_lang[str(language)][:]
# 'SERVER_LANGUAGE' is an indicator for this script to pick
# a server in different language.
custom_server_lang = scenario_json.get(
'SERVER_LANGUAGE', None)
custom_client_lang = scenario_json.get(
'CLIENT_LANGUAGE', None)
scenario_json = scenario_config.remove_nonproto_fields(
scenario_json)
if custom_server_lang and custom_client_lang:
raise Exception(
'Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE'
'in the same scenario')
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
print('Warning: Skipping scenario %s as' %
scenario_json['name'])
print(
'SERVER_LANGUAGE is set to %s yet the language has '
'not been selected with -l' %
custom_server_lang)
continue
for idx in range(0, scenario_json['num_servers']):
# replace first X workers by workers of a different language
workers[idx] = workers_by_lang[custom_server_lang][
idx]
if custom_client_lang:
if not workers_by_lang.get(custom_client_lang, []):
print('Warning: Skipping scenario %s as' %
scenario_json['name'])
print(
'CLIENT_LANGUAGE is set to %s yet the language has '
'not been selected with -l' %
custom_client_lang)
continue
for idx in range(scenario_json['num_servers'],
len(workers)):
# replace all client workers by workers of a different language,
# leave num_server workers as they are server workers.
workers[idx] = workers_by_lang[custom_client_lang][
idx]
scenario = Scenario(
create_scenario_jobspec(
scenario_json, [w.host_and_port for w in workers],
remote_host=remote_host,
bq_result_table=bq_result_table,
server_cpu_load=server_cpu_load), workers,
scenario_json['name'])
scenarios.append(scenario)
return scenarios
def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
category='all', bq_result_table=None,
netperf=False, netperf_hosts=[], server_cpu_load=0):
"""Create jobspecs for scenarios to run."""
all_workers = [worker
for workers in workers_by_lang.values()
for worker in workers]
scenarios = []
_NO_WORKERS = []
if netperf:
if not netperf_hosts:
netperf_server='localhost'
netperf_client=None
elif len(netperf_hosts) == 1:
netperf_server=netperf_hosts[0]
netperf_client=netperf_hosts[0]
else:
netperf_server=netperf_hosts[0]
netperf_client=netperf_hosts[1]
scenarios.append(Scenario(
create_netperf_jobspec(server_host=netperf_server,
client_host=netperf_client,
bq_result_table=bq_result_table),
_NO_WORKERS, 'netperf'))
for language in languages:
for scenario_json in language.scenarios():
if re.search(args.regex, scenario_json['name']):
categories = scenario_json.get('CATEGORIES', ['scalable', 'smoketest'])
if category in categories or category == 'all':
workers = workers_by_lang[str(language)][:]
# 'SERVER_LANGUAGE' is an indicator for this script to pick
# a server in different language.
custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None)
custom_client_lang = scenario_json.get('CLIENT_LANGUAGE', None)
scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
if custom_server_lang and custom_client_lang:
raise Exception('Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE'
'in the same scenario')
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
print('Warning: Skipping scenario %s as' % scenario_json['name'])
print('SERVER_LANGUAGE is set to %s yet the language has '
'not been selected with -l' % custom_server_lang)
continue
for idx in range(0, scenario_json['num_servers']):
# replace first X workers by workers of a different language
workers[idx] = workers_by_lang[custom_server_lang][idx]
if custom_client_lang:
if not workers_by_lang.get(custom_client_lang, []):
print('Warning: Skipping scenario %s as' % scenario_json['name'])
print('CLIENT_LANGUAGE is set to %s yet the language has '
'not been selected with -l' % custom_client_lang)
continue
for idx in range(scenario_json['num_servers'], len(workers)):
# replace all client workers by workers of a different language,
# leave num_server workers as they are server workers.
workers[idx] = workers_by_lang[custom_client_lang][idx]
scenario = Scenario(
create_scenario_jobspec(scenario_json,
[w.host_and_port for w in workers],
remote_host=remote_host,
bq_result_table=bq_result_table,
server_cpu_load=server_cpu_load),
workers,
scenario_json['name'])
scenarios.append(scenario)
return scenarios
def _scenario_json_string(scenario_json, is_tsan):
scenarios_json = {'scenarios': [scenario_config.remove_nonproto_fields(mutate_scenario(scenario_json, is_tsan))]}
return json.dumps(scenarios_json)
def _scenario_json_string(scenario_json):
return json.dumps(scenario_config.remove_nonproto_fields(scenario_json))
def create_scenarios(
languages,
workers_by_lang,
remote_host=None,
regex=".*",
category="all",
bq_result_table=None,
netperf=False,
netperf_hosts=[],
):
"""Create jobspecs for scenarios to run."""
all_workers = [worker for workers in workers_by_lang.values() for worker in workers]
scenarios = []
if netperf:
if not netperf_hosts:
netperf_server = "localhost"
netperf_client = None
elif len(netperf_hosts) == 1:
netperf_server = netperf_hosts[0]
netperf_client = netperf_hosts[0]
else:
netperf_server = netperf_hosts[0]
netperf_client = netperf_hosts[1]
scenarios.append(
create_netperf_jobspec(
server_host=netperf_server, client_host=netperf_client, bq_result_table=bq_result_table
)
)
for language in languages:
for scenario_json in language.scenarios():
if re.search(args.regex, scenario_json["name"]):
categories = scenario_json.get("CATEGORIES", [])
if category in categories or (category == "all" and categories != ["sweep"]):
workers = workers_by_lang[str(language)]
# 'SERVER_LANGUAGE' is an indicator for this script to pick
# a server in different language.
custom_server_lang = scenario_json.get("SERVER_LANGUAGE", None)
custom_client_lang = scenario_json.get("CLIENT_LANGUAGE", None)
scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
if custom_server_lang and custom_client_lang:
raise Exception(
"Cannot set both custom CLIENT_LANGUAGE and SERVER_LANGUAGE" "in the same scenario"
)
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
print("Warning: Skipping scenario %s as" % scenario_json["name"])
print(
"SERVER_LANGUAGE is set to %s yet the language has "
"not been selected with -l" % custom_server_lang
)
continue
for idx in range(0, scenario_json["num_servers"]):
# replace first X workers by workers of a different language
workers[idx] = workers_by_lang[custom_server_lang][idx]
if custom_client_lang:
if not workers_by_lang.get(custom_client_lang, []):
print("Warning: Skipping scenario %s as" % scenario_json["name"])
print(
"CLIENT_LANGUAGE is set to %s yet the language has "
"not been selected with -l" % custom_client_lang
)
continue
for idx in range(scenario_json["num_servers"], len(workers)):
# replace all client workers by workers of a different language,
# leave num_server workers as they are server workers.
workers[idx] = workers_by_lang[custom_client_lang][idx]
scenario = create_scenario_jobspec(
scenario_json, workers, remote_host=remote_host, bq_result_table=bq_result_table
)
scenarios.append(scenario)
# the very last scenario requests shutting down the workers.
scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host))
return scenarios
def _scenario_json_string(scenario_json): return json.dumps(scenario_config.remove_nonproto_fields(scenario_json))
