def create_source_file_map(cspec):
logging.info('CREATING source file map')
source_file_map = {}
#need to add trials in with cspec.permuters before expanding
trials_list = cspec.get_trials_list()
permuters_with_trials = {}
for key, val in cspec.permuters.items():
permuters_with_trials[key] = val
permuters_with_trials['trial'] = trials_list
permutation_list = permutations.expand_permutations(permuters_with_trials)
for permutation_info in permutation_list:
pcode = permutations.generate_perm_code(permutation_info,
cspec.concise_print_map,
permutations.INCLUDE_TRIALS)
#print "permutation_code {0}".format(permutation_code)
permutation_output_dir = cspec.generate_results_dir_for_permutation(
pcode)
from_file_path_with_results_dir_resolved = "{0}/{1}".format(
permutation_output_dir, cspec.output_filename)
#from_file_path_with_results_dir_resolved = cspec.scores_from_filepath.replace('<permutation_output_dir>',permutation_output_dir)
#print "from_file_path_with_results_dir_resolved {0}".format(from_file_path_with_results_dir_resolved)
scores_permutations_list = permutations.expand_permutations(
cspec.scores_permuters)
if (len(scores_permutations_list) == 0):
fully_resolved_from_filepath = get_fully_resolved_from_filepath(
from_file_path_with_results_dir_resolved, permutation_info,
cspec, {})
full_perm_code = get_full_perm_code(permutation_info, {}, cspec)
source_file_map[full_perm_code] = fully_resolved_from_filepath
else:
for scores_permutations_info in scores_permutations_list:
fully_resolved_from_filepath = get_fully_resolved_from_filepath(
from_file_path_with_results_dir_resolved, permutation_info,
cspec, scores_permutations_info)
full_perm_code = get_full_perm_code(permutation_info,
scores_permutations_info,
cspec)
source_file_map[full_perm_code] = fully_resolved_from_filepath
#print "source_file_map[{0}] = {1}".format(full_perm_code, fully_resolved_from_filepath)
return source_file_map
def __init__(self, cspec, stdout):
self.cspec = cspec
self.job_number_for_perm_code_map = {}
self.perm_code_for_job_number_map = {}
self.cluster_script_for_perm_code_map = {}
self.perm_info_for_perm_code_map = {}
self.perm_code_for_perm_info_map = {}
self.result_dir_for_perm_code_map = {}
self.permuters_including_trials = cspec.get_permuters_trials_included()
self.permutation_info_list_full = permutations.expand_permutations(
self.permuters_including_trials)
self.run_perm_codes_list = []
self.job_num_width = self.get_job_number_width(
self.permutation_info_list_full)
# associate job numbers here and lookup from then on
user_job_number = 1
if cspec.first_job_number != '':
user_job_number = int(cspec.first_job_number)
for perm_info in self.permutation_info_list_full:
perm_code = permutations.generate_perm_code(
perm_info, cspec.concise_print_map,
permutations.INCLUDE_TRIALS)
self.run_perm_codes_list.append(perm_code)
user_job_number_as_string = get_formatted_user_job_number(
user_job_number, self.job_num_width)
self.job_number_for_perm_code_map[
perm_code] = user_job_number_as_string
self.perm_code_for_job_number_map[
user_job_number_as_string] = perm_code
self.perm_info_for_perm_code_map[perm_code] = perm_info
perm_info_string = "{0}".format(perm_info)
self.perm_code_for_perm_info_map[perm_info_string] = perm_code
resolved_results_dir = permutations.get_resolved_results_dir_for_permutation(
perm_info, cspec)
self.result_dir_for_perm_code_map[perm_code] = resolved_results_dir
cscript = cluster_script.ClusterScript(user_job_number_as_string,
perm_info, cspec,
perm_info['trial'], stdout)
self.cluster_script_for_perm_code_map[perm_code] = cscript
user_job_number = user_job_number + 1
def persist(self):
cspec = self.cspec
# generate the column names
ft = open(self.target_timings_path, 'w')
print "persisting {0}".format(self.target_timings_path)
# scores_y_axis:letter
# scores_x_axis:number,animal
x_permuters = {}
y_permuters = {}
for item in cspec.scores_x_axis:
x_permuters[item] = cspec.permuters[item]
for item in cspec.scores_y_axis:
y_permuters[item] = cspec.permuters[item]
x_permutations = permutations.expand_permutations(x_permuters)
y_permutations = permutations.expand_permutations(y_permuters)
# make a list of x permutation codes for use later
x_perm_codes = []
for x_permutation in x_permutations:
x_perm_codes.append(
permutations.generate_perm_code(x_permutation,
cspec.concise_print_map,
permutations.IGNORE_TRIALS))
medians = {}
# write the x_axis column names
header = "{0},".format(
pooled_results_file.beautify_header("{0}".format(
cspec.scores_y_axis)))
for x_permutation in x_permutations:
concise_x_permutation = permutations.generate_perm_code(
x_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
header = "{0}{1},".format(header, concise_x_permutation)
header = header.rstrip(',')
ft.write("{0}\n".format(header))
for y_permutation in y_permutations:
concise_y_permutation = permutations.generate_perm_code(
y_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
timings_line = "{0},".format(concise_y_permutation)
for x_permutation in x_permutations:
trials_list = cspec.get_trials_list()
trial_timing_values = []
for trial in trials_list:
cluster_job_perm_code = gen_cluster_job_perm_code_from_pieces(
y_permutation, x_permutation,
self.filename_permutation_info, cspec, trial)
timing_value = get_timing_value_for_run(
cluster_job_perm_code, self.cluster_runs, self.stdout)
#print 'timing_value : {0}'.format(timing_value)
trial_timing_values.append(timing_value)
median_timing = pooled_results_file.get_median(
trial_timing_values, True)
#print 'median_timing {0}'.format(median_timing)
timings_line = "{0}{1},".format(timings_line, median_timing)
x_perm_code = permutations.generate_perm_code(
x_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
pooled_results_file.record_median(x_perm_code, medians,
median_timing)
timings_line.rstrip(',')
ft.write("{0}\n".format(timings_line))
# add the averages line
line = "averages,"
for x_perm_code in x_perm_codes:
medians_list = medians[x_perm_code]
average = pooled_results_file.compute_average_medians(
medians_list, True)
line = "{0}{1},".format(line, average)
line = line.rstrip(',')
ft.write("{0}\n".format(line))
ft.close()
def test_expand_permutations(self):
stdout = mock_stdout.MockStdout()
lines = []
lines.append("#pspec\n")
lines.append("(permute):number=range(1,4)\n")
lines.append("(permute):letter=AAA,BBB\n")
lines.append("(permute):singleton_val=300\n")
lines.append("(permute):animal=dog,cat\n")
cspec = cluster_spec.ClusterSpec("/foo/bar/baz.cspec", lines, stdout,
[], False, False)
permuters = cspec.permuters
dict_list = permutations.expand_permutations(permuters)
#print dict_list
self.assertTrue(len(dict_list) == 12)
dict1 = {
'animal': 'cat',
'number': '1',
'letter': 'AAA',
'singleton_val': '300'
}
dict2 = {
'animal': 'cat',
'number': '1',
'letter': 'BBB',
'singleton_val': '300'
}
dict3 = {
'animal': 'cat',
'number': '2',
'letter': 'AAA',
'singleton_val': '300'
}
dict4 = {
'animal': 'cat',
'number': '2',
'letter': 'BBB',
'singleton_val': '300'
}
dict5 = {
'animal': 'cat',
'number': '3',
'letter': 'AAA',
'singleton_val': '300'
}
dict6 = {
'animal': 'cat',
'number': '3',
'letter': 'BBB',
'singleton_val': '300'
}
dict7 = {
'animal': 'dog',
'number': '1',
'letter': 'AAA',
'singleton_val': '300'
}
dict8 = {
'animal': 'dog',
'number': '1',
'letter': 'BBB',
'singleton_val': '300'
}
dict9 = {
'animal': 'dog',
'number': '2',
'letter': 'AAA',
'singleton_val': '300'
}
dict10 = {
'animal': 'dog',
'number': '2',
'letter': 'BBB',
'singleton_val': '300'
}
dict11 = {
'animal': 'dog',
'number': '3',
'letter': 'AAA',
'singleton_val': '300'
}
dict12 = {
'animal': 'dog',
'number': '3',
'letter': 'BBB',
'singleton_val': '300'
}
self.assertTrue(dict1 in dict_list)
self.assertTrue(dict2 in dict_list)
self.assertTrue(dict3 in dict_list)
self.assertTrue(dict4 in dict_list)
self.assertTrue(dict5 in dict_list)
self.assertTrue(dict6 in dict_list)
self.assertTrue(dict7 in dict_list)
self.assertTrue(dict8 in dict_list)
self.assertTrue(dict9 in dict_list)
self.assertTrue(dict10 in dict_list)
self.assertTrue(dict11 in dict_list)
self.assertTrue(dict12 in dict_list)
def persist(self):
cspec = self.cspec
# generate the column names
f = open(self.target_path, 'w')
self.stdout.println("persisting {0}".format(self.target_path))
# scores_y_axis:letter
# scores_x_axis:number,animal
x_permuters = {}
y_permuters = {}
for item in cspec.scores_x_axis:
x_permuters[item] = cspec.permuters[item]
for item in cspec.scores_y_axis:
y_permuters[item] = cspec.permuters[item]
x_permutations = permutations.expand_permutations(x_permuters)
y_permutations = permutations.expand_permutations(y_permuters)
# write the x_axis column names
header = "{0},".format(
beautify_header("{0}".format(cspec.scores_y_axis)))
for x_permutation in x_permutations:
concise_x_permutation = permutations.generate_perm_code(
x_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
header = "{0}{1},".format(header, concise_x_permutation)
header = header.rstrip(',')
f.write("{0}\n".format(header))
medians = {}
# make a list of x permutation codes for use later
x_perm_codes = []
for x_permutation in x_permutations:
x_perm_codes.append(
permutations.generate_perm_code(x_permutation,
cspec.concise_print_map,
permutations.IGNORE_TRIALS))
# the main loop
for y_permutation in y_permutations:
concise_y_permutation = permutations.generate_perm_code(
y_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
self.stdout.println('y axis: {0}'.format(concise_y_permutation))
line = "{0},".format(concise_y_permutation)
for x_permutation in x_permutations:
trials_list = cspec.get_trials_list()
trial_values = []
for trial in trials_list:
result_file_perm_code = gen_result_perm_code_from_pieces(
y_permutation, x_permutation,
self.filename_permutation_info, cspec, trial)
#print "self.source_file_map {0}".format(self.source_file_map)
source_file_path = self.source_file_map[
result_file_perm_code]
#print "SOURCE_FILE_PATH : {0}".format(source_file_path)
value = get_result_from_file(source_file_path,
cspec.scores_from_colname,
cspec.scores_from_rownum)
trial_values.append(value)
median_value = get_median(trial_values, False)
line = "{0}{1},".format(line, median_value)
x_perm_code = permutations.generate_perm_code(
x_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
record_median(x_perm_code, medians, median_value)
line = line.rstrip(',')
f.write("{0}\n".format(line))
# add the averages line
line = "averages,"
for x_perm_code in x_perm_codes:
medians_list = medians[x_perm_code]
average = compute_average_medians(medians_list, False)
line = "{0}{1},".format(line, average)
line = line.rstrip(',')
f.write("{0}\n".format(line))
f.close()