def run_tests(actual_objective_functions, model_name, dir_name, cluster, should_log, compiler_name, frequency_str=''): expected_objective_functions = common_code.csv_to_dict( '%s/bamboo/integration_tests/expected_values/%s/%s/expected_%s_objective_functions.csv' % (dir_name, cluster, compiler_name, model_name)) errors = [] all_values = [] tolerance = 0.05 # Are we within tolerance * expected_value? outside_tolerance = lambda x, y: abs(x - y) > abs(tolerance * y) error_if(outside_tolerance, '!=', 'training_objective_function', actual_objective_functions, expected_objective_functions, model_name, errors, all_values, frequency_str) print('Errors for: %s %s (%d)' % (model_name, compiler_name, len(errors))) for error in errors: print(error) if should_log: print('All values for: %s %s (%d)' % (model_name, compiler_name, len(all_values))) for value in all_values: print(value) assert errors == []
def run_tests(actual_performance, model_name, dir_name, should_log, compiler_name, cluster, frequency_str=''): expected_performance = common_code.csv_to_dict( '%s/bamboo/integration_tests/expected_values/%s/%s/expected_performance.csv' % (dir_name, cluster, compiler_name)) errors = [] all_values = [] greater_than = operator.gt less_than = operator.lt max_run_time = error_if(greater_than, '>', 'training_run_time', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) max_mean = error_if(greater_than, '>', 'training_mean', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) max_max = error_if(greater_than, '>', 'training_max', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) max_min = error_if(greater_than, '>', 'training_min', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) max_stdev = error_if(greater_than, '>', 'training_stdev', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) min_accuracy = error_if(less_than, '<', 'test_accuracy', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) if os.environ['LOGNAME'] == 'lbannusr': key = 'bamboo_planKey' if key in os.environ: plan = os.environ[key] if plan in ['LBANN-NIGHTD', 'LBANN-WD']: archive_file = '/usr/workspace/wsb/lbannusr/archives/%s/%s/%s/performance_%s.txt' % ( plan, cluster, compiler_name, model_name) with open(archive_file, 'a') as archive: archive.write( '%s, %f, %f, %f, %f, %f, %f\n' % (os.environ['bamboo_buildNumber'], max_run_time, max_mean, max_max, max_min, max_stdev, min_accuracy)) else: print('The plan %s does not have archiving activated' % plan) else: print('%s is not in os.environ' % key) else: print('os.environ["LOGNAME"]=%s' % os.environ['LOGNAME']) print('Errors for: %s (%d)' % (model_name, len(errors))) for error in errors: print(error) if should_log: print('All values for: %s (%d)' % (model_name, len(all_values))) for value in all_values: print(value) assert errors == []
def run_tests(actual_objective_functions, model_name, dir_name, cluster, should_log, compiler_name, frequency_str=''): expected_objective_functions = common_code.csv_to_dict('%s/bamboo/integration_tests/expected_values/%s/%s/expected_%s_objective_functions.csv' % (dir_name, cluster, compiler_name, model_name)) errors = [] all_values = [] tolerance = 0.05 # Are we within tolerance * expected_value? outside_tolerance = lambda x,y: abs(x - y) > abs(tolerance * y) error_if(outside_tolerance, '!=', 'training_objective_function', actual_objective_functions, expected_objective_functions, model_name, errors, all_values, frequency_str) print('Errors for: %s %s (%d)' % (model_name, compiler_name, len(errors))) for error in errors: print(error) if should_log: print('All values for: %s %s (%d)' % (model_name, compiler_name, len(all_values))) for value in all_values: print(value) assert errors == []
def run_tests(actual_performance, model_name, dir_name, should_log, compiler_name, cluster, frequency_str=''): expected_performance = common_code.csv_to_dict('%s/bamboo/integration_tests/expected_values/%s/%s/expected_performance.csv' % (dir_name, cluster, compiler_name)) errors = [] all_values = [] greater_than = operator.gt less_than = operator.lt max_run_time = error_if(greater_than, '>', 'training_run_time', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) max_mean = error_if(greater_than, '>', 'training_mean', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) max_max = error_if(greater_than, '>', 'training_max', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) max_min = error_if(greater_than, '>', 'training_min', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) max_stdev = error_if(greater_than, '>', 'training_stdev', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) min_accuracy = error_if(less_than, '<', 'test_accuracy', actual_performance, expected_performance, model_name, errors, all_values, frequency_str) if os.environ['LOGNAME'] == 'lbannusr': key = 'bamboo_planKey' if key in os.environ: plan = os.environ[key] if plan in ['LBANN-NIGHTD', 'LBANN-WD']: archive_file = '/usr/workspace/wsb/lbannusr/archives/%s/%s/%s/performance_%s.txt' % (plan, cluster, compiler_name, model_name) with open(archive_file, 'a') as archive: archive.write('%s, %f, %f, %f, %f, %f, %f\n' % (os.environ['bamboo_buildNumber'], max_run_time, max_mean, max_max, max_min, max_stdev, min_accuracy)) else: print('The plan %s does not have archiving activated' % plan) else: print('%s is not in os.environ' % key) else: print('os.environ["LOGNAME"]=%s' % os.environ['LOGNAME']) print('Errors for: %s %s (%d)' % (model_name, compiler_name, len(errors))) for error in errors: print(error) if should_log: print('All values for: %s %s (%d)' % (model_name, compiler_name, len(all_values))) for value in all_values: print(value) assert errors == []