def run_tests(actual_objective_functions,
model_name,
dir_name,
cluster,
should_log,
compiler_name,
frequency_str=''):
expected_objective_functions = common_code.csv_to_dict(
'%s/bamboo/integration_tests/expected_values/%s/%s/expected_%s_objective_functions.csv'
% (dir_name, cluster, compiler_name, model_name))
errors = []
all_values = []
tolerance = 0.05
# Are we within tolerance * expected_value?
outside_tolerance = lambda x, y: abs(x - y) > abs(tolerance * y)
error_if(outside_tolerance, '!=', 'training_objective_function',
actual_objective_functions, expected_objective_functions,
model_name, errors, all_values, frequency_str)
print('Errors for: %s %s (%d)' % (model_name, compiler_name, len(errors)))
for error in errors:
print(error)
if should_log:
print('All values for: %s %s (%d)' %
(model_name, compiler_name, len(all_values)))
for value in all_values:
print(value)
assert errors == []
def run_tests(actual_performance,
model_name,
dir_name,
should_log,
compiler_name,
cluster,
frequency_str=''):
expected_performance = common_code.csv_to_dict(
'%s/bamboo/integration_tests/expected_values/%s/%s/expected_performance.csv'
% (dir_name, cluster, compiler_name))
errors = []
all_values = []
greater_than = operator.gt
less_than = operator.lt
max_run_time = error_if(greater_than, '>', 'training_run_time',
actual_performance, expected_performance,
model_name, errors, all_values, frequency_str)
max_mean = error_if(greater_than, '>', 'training_mean', actual_performance,
expected_performance, model_name, errors, all_values,
frequency_str)
max_max = error_if(greater_than, '>', 'training_max', actual_performance,
expected_performance, model_name, errors, all_values,
frequency_str)
max_min = error_if(greater_than, '>', 'training_min', actual_performance,
expected_performance, model_name, errors, all_values,
frequency_str)
max_stdev = error_if(greater_than, '>', 'training_stdev',
actual_performance, expected_performance, model_name,
errors, all_values, frequency_str)
min_accuracy = error_if(less_than, '<', 'test_accuracy',
actual_performance, expected_performance,
model_name, errors, all_values, frequency_str)
if os.environ['LOGNAME'] == 'lbannusr':
key = 'bamboo_planKey'
if key in os.environ:
plan = os.environ[key]
if plan in ['LBANN-NIGHTD', 'LBANN-WD']:
archive_file = '/usr/workspace/wsb/lbannusr/archives/%s/%s/%s/performance_%s.txt' % (
plan, cluster, compiler_name, model_name)
with open(archive_file, 'a') as archive:
archive.write(
'%s, %f, %f, %f, %f, %f, %f\n' %
(os.environ['bamboo_buildNumber'], max_run_time,
max_mean, max_max, max_min, max_stdev, min_accuracy))
else:
print('The plan %s does not have archiving activated' % plan)
else:
print('%s is not in os.environ' % key)
else:
print('os.environ["LOGNAME"]=%s' % os.environ['LOGNAME'])
print('Errors for: %s (%d)' % (model_name, len(errors)))
for error in errors:
print(error)
if should_log:
print('All values for: %s (%d)' % (model_name, len(all_values)))
for value in all_values:
print(value)
assert errors == []
def run_tests(actual_objective_functions, model_name, dir_name, cluster, should_log, compiler_name, frequency_str=''):
expected_objective_functions = common_code.csv_to_dict('%s/bamboo/integration_tests/expected_values/%s/%s/expected_%s_objective_functions.csv' % (dir_name, cluster, compiler_name, model_name))
errors = []
all_values = []
tolerance = 0.05
# Are we within tolerance * expected_value?
outside_tolerance = lambda x,y: abs(x - y) > abs(tolerance * y)
error_if(outside_tolerance, '!=', 'training_objective_function', actual_objective_functions, expected_objective_functions, model_name, errors, all_values, frequency_str)
print('Errors for: %s %s (%d)' % (model_name, compiler_name, len(errors)))
for error in errors:
print(error)
if should_log:
print('All values for: %s %s (%d)' % (model_name, compiler_name, len(all_values)))
for value in all_values:
print(value)
assert errors == []
def run_tests(actual_performance, model_name, dir_name, should_log, compiler_name, cluster, frequency_str=''):
expected_performance = common_code.csv_to_dict('%s/bamboo/integration_tests/expected_values/%s/%s/expected_performance.csv' % (dir_name, cluster, compiler_name))
errors = []
all_values = []
greater_than = operator.gt
less_than = operator.lt
max_run_time = error_if(greater_than, '>', 'training_run_time', actual_performance, expected_performance, model_name, errors, all_values, frequency_str)
max_mean = error_if(greater_than, '>', 'training_mean', actual_performance, expected_performance, model_name, errors, all_values, frequency_str)
max_max = error_if(greater_than, '>', 'training_max', actual_performance, expected_performance, model_name, errors, all_values, frequency_str)
max_min = error_if(greater_than, '>', 'training_min', actual_performance, expected_performance, model_name, errors, all_values, frequency_str)
max_stdev = error_if(greater_than, '>', 'training_stdev', actual_performance, expected_performance, model_name, errors, all_values, frequency_str)
min_accuracy = error_if(less_than, '<', 'test_accuracy', actual_performance, expected_performance, model_name, errors, all_values, frequency_str)
if os.environ['LOGNAME'] == 'lbannusr':
key = 'bamboo_planKey'
if key in os.environ:
plan = os.environ[key]
if plan in ['LBANN-NIGHTD', 'LBANN-WD']:
archive_file = '/usr/workspace/wsb/lbannusr/archives/%s/%s/%s/performance_%s.txt' % (plan, cluster, compiler_name, model_name)
with open(archive_file, 'a') as archive:
archive.write('%s, %f, %f, %f, %f, %f, %f\n' % (os.environ['bamboo_buildNumber'], max_run_time, max_mean, max_max, max_min, max_stdev, min_accuracy))
else:
print('The plan %s does not have archiving activated' % plan)
else:
print('%s is not in os.environ' % key)
else:
print('os.environ["LOGNAME"]=%s' % os.environ['LOGNAME'])
print('Errors for: %s %s (%d)' % (model_name, compiler_name, len(errors)))
for error in errors:
print(error)
if should_log:
print('All values for: %s %s (%d)' % (model_name, compiler_name, len(all_values)))
for value in all_values:
print(value)
assert errors == []