def warm_up():
"""Loads and warms up the tensor features model."""
global DESCRIPTION_HANDLER, TENSOR_MODEL, TENSOR_CONFIG, WARMED_UP
if WARMED_UP:
return
WARMED_UP = True
# Use the default choices for the description handler and tensor model. The
# Colab interface will not allow users to change these.
DESCRIPTION_HANDLER = description_handler_factory.create_handler(
DEFAULT_SETTINGS.description_handler_name)
if (not DEFAULT_SETTINGS.tensor_model.config_path or
not DEFAULT_SETTINGS.tensor_model.checkpoint_path):
return
try:
TENSOR_CONFIG = tensor_features_model.load_config(
DEFAULT_SETTINGS.tensor_model.config_path)
TENSOR_MODEL = tensor_features_model.get_model(TENSOR_CONFIG)
tensor_checkpoint = tensor_features_model.create_checkpoint(TENSOR_MODEL)
tensor_checkpoint.restore(
DEFAULT_SETTINGS.tensor_model.checkpoint_path).expect_partial()
# Warm up. Running the model for the first time takes an extra ~10 seconds.
value_search.operation_multipliers_from_tensor_model(
all_benchmarks.find_benchmark_with_name('simple_cast'),
TENSOR_MODEL, TENSOR_CONFIG, DEFAULT_SETTINGS)
except Exception: # pylint: disable=broad-except
# No matter what goes wrong with loading the tensor features model, we
# should fall back to value search without the model.
print('Could not load the tensor features model. ' + CONTACT_MESSAGE)
TENSOR_MODEL = None
TENSOR_CONFIG = None
def test_value_search_can_load_data(self, mock_stdout):
benchmark = all_benchmarks.find_benchmark_with_name('simple_cast')
handler = description_handler_factory.create_handler('naive_bayes')
settings = settings_module.from_dict({
'timeout':
5,
'printing.prioritized_operations':
True,
'printing.deprioritized_operations':
True,
})
tensor_config = tensor_features_model.load_config(
settings.tensor_model.config_path)
tensor_model = tensor_features_model.get_model(tensor_config)
tensor_checkpoint = tensor_features_model.create_checkpoint(
tensor_model)
tensor_checkpoint.restore(
settings.tensor_model.checkpoint_path).expect_partial()
results = value_search.run_value_search(benchmark=benchmark,
description_handler=handler,
settings=settings,
tensor_model=tensor_model,
tensor_config=tensor_config)
self.assertLen(results.solutions, 1)
self.assertIn('BOW handler prioritized tf.cast(x, dtype)',
mock_stdout.getvalue())
self.assertIn('Tensor features model prioritized tf.cast(x, dtype)',
mock_stdout.getvalue())
def test_find_benchmark_with_name(self, benchmark_name, expect_none):
result = all_benchmarks.find_benchmark_with_name(
benchmark_name, include_ignored=True, modules=[test_benchmarks])
if expect_none:
self.assertIsNone(result)
else:
self.assertIsNotNone(result)
self.assertEqual(result.name, benchmark_name)
def test_run_value_search_works_for_small_max_weight(
self, benchmark_name, max_weight):
benchmark = all_benchmarks.find_benchmark_with_name(benchmark_name)
results = value_search.run_value_search(
benchmark=benchmark,
settings=settings_module.from_dict({'max_weight': max_weight}))
print("Note: if you're inspecting the test log, the above failure is "
"expected in test_run_value_search_works_for_small_max_weight.")
self.assertEmpty(results.solutions)
def main(unused_argv):
settings = settings_module.from_list(FLAGS.settings)
description_handler = description_handler_factory.create_handler(
settings.description_handler_name)
benchmark = all_benchmarks.find_benchmark_with_name(FLAGS.benchmark_name)
if not benchmark:
raise ValueError('Unknown benchmark: {}'.format(FLAGS.benchmark_name))
compute_search_space_size(benchmark=benchmark,
settings=settings,
description_handler=description_handler)
def test_run_value_search_works_for_important_benchmarks(
self, benchmark_name, timeout, solution):
# These benchmarks are frequently used in demos. We should ensure that they
# are solved quickly and consistently. While the exact solution may change
# as the algorithm improves, we should ensure that the solution is always
# simple and understandable.
handler = description_handler_factory.create_handler('tfidf')
benchmark = all_benchmarks.find_benchmark_with_name(benchmark_name)
results = value_search.run_value_search(
benchmark=benchmark,
description_handler=handler,
settings=settings_module.from_dict({'timeout': timeout}))
print('Time for benchmark {}: {:.2f} sec'.format(
benchmark_name, results.total_time))
self.assertLen(results.solutions, 1)
self.assertEqual(results.solutions[0].expression, solution)
def run_on_all_benchmarks():
"""Runs value search on all benchmarks, printing results to stdout."""
benchmark_count = 0
benchmark_success = 0
unsolved_benchmarks = []
solution_times = [] # Only including successful tasks.
settings = settings_module.from_list(FLAGS.settings)
description_handler = description_handler_factory.create_handler(
settings.description_handler_name)
print('Description handler: {!r}\n'.format(description_handler))
results_json = {
'benchmark_name': FLAGS.benchmark_name,
'settings': settings.as_dict(),
'notes': FLAGS.notes,
'results': [],
}
if (settings.tensor_model.config_path and
settings.tensor_model.checkpoint_path):
tensor_config = tensor_features_model.load_config(
settings.tensor_model.config_path)
tensor_model = tensor_features_model.get_model(tensor_config)
checkpoint = tensor_features_model.create_checkpoint(tensor_model)
checkpoint.restore(settings.tensor_model.checkpoint_path).expect_partial()
# Warm up. Running the model for the first time takes an extra ~10 seconds.
print('Warming up the tensor features model...')
value_search.operation_multipliers_from_tensor_model(
all_benchmarks.find_benchmark_with_name('simple_cast'),
tensor_model, tensor_config, settings)
print('Finished warming up.')
else:
tensor_config = None
tensor_model = None
print('=' * 80)
modules = [google_benchmarks, stackoverflow_benchmarks]
for benchmark in all_benchmarks.get_chosen_benchmarks(
FLAGS.benchmark_name, modules=modules):
gc.collect()
print('Performing value search for benchmark {}.\n'
.format(benchmark.name))
benchmark_count += 1
result = value_search.run_value_search(
benchmark=benchmark,
settings=settings,
description_handler=description_handler,
tensor_model=tensor_model,
tensor_config=tensor_config)
if settings.printing.statistics:
print('\nOperation statistics:\n{}'.format(
result.statistics.statistics_as_string(
num_unique_values=len(result.value_set),
elapsed_time=result.total_time,
sort_by_time=settings.printing.statistics_sort_by_time)))
solutions = result.solutions
if solutions:
first_solution = solutions[0]
print('\nBest solution of weight {} found in {:.2f} sec:\n{}'.format(
first_solution.weight, first_solution.time,
first_solution.expression))
benchmark_success += 1
solution_times.append(first_solution.time)
else:
unsolved_benchmarks.append(benchmark)
print('=' * 80)
sys.stdout.flush()
results_json['results'].append({
'name': benchmark.name,
'solved': bool(solutions),
'solution': solutions[0].expression if solutions else None,
'solution_weight': solutions[0].weight if solutions else None,
'time': solutions[0].time if solutions else None,
})
solve_time_total = sum(solution_times)
solve_time_mean = np.mean(solution_times)
solve_time_geometric_mean = mstats.gmean(solution_times)
results_json['num_benchmarks'] = benchmark_count
results_json['num_solved'] = benchmark_success
results_json['solve_time_total'] = solve_time_total
results_json['solve_time_mean'] = solve_time_mean
results_json['solve_time_geometric_mean'] = solve_time_geometric_mean
print('Solved {} out of {} benchmarks in {:.2f} sec.'.format(
benchmark_success, benchmark_count, solve_time_total))
print('\n'
'Arithmetic mean of solve times: {:.2f} sec\n'
'Geometric mean of solve times: {:.2f} sec\n'.format(
solve_time_mean, solve_time_geometric_mean))
print('Unsolved benchmarks:')
for unsolved in unsolved_benchmarks:
print('Name: {}, target program: {}'.format(
unsolved.name, unsolved.target_program))
print()
if FLAGS.json_output and FLAGS.benchmark_name == 'ALL':
with open(FLAGS.json_output, 'w') as json_file:
json.dump(results_json, json_file,
indent=4, sort_keys=True, separators=(',', ': '))
json_file.write('\n')
print('Wrote JSON results to {}.'.format(FLAGS.json_output))
else:
print('Did not write JSON results file.')
def benchmark_name_validator(benchmark_name):
"""Checks that benchmark_name is "ALL", or refers to exactly one benchmark."""
return (benchmark_name == 'ALL' or
all_benchmarks.find_benchmark_with_name(benchmark_name) is not None)
