def test_generate_perm_code(self):
permutation_info = {'number': '1', 'letter': 'AAA', 'trial': '1'}
concisePrintMap = {'number': 'n', 'AAA': 'A'}
code = permutations.generate_perm_code(permutation_info,
concisePrintMap,
permutations.IGNORE_TRIALS)
self.assertTrue(code == 'letter_A_n_1')
code = permutations.generate_perm_code(permutation_info,
concisePrintMap,
permutations.INCLUDE_TRIALS)
self.assertTrue(code == 'letter_A_n_1_trial_1')
def __init__(self, user_job_number, permutation_info, cspec, trial,
stdout):
'''
Constructor
'''
#self.cluster_system = cluster_system
self.stdout = stdout
self.cspec = cspec
self.trial = trial
self.permutation_info = permutation_info
self.pcode = permutations.generate_perm_code(
permutation_info, cspec.concise_print_map,
permutations.INCLUDE_TRIALS)
self.key_val_map = {}
for key, val in cspec.key_val_map.items():
self.key_val_map[key] = val
self.type = "permutation_driver_file"
self.resolved_results_dir = permutations.get_resolved_results_dir_for_permutation(
permutation_info, cspec)
#print "SETTING resolved_results_dir : {0}".format(self.resolved_results_dir)
self.key_val_map['permutation_output_dir'] = self.resolved_results_dir
self.user_job_number = user_job_number
self.script_dir = cspec.script_dir
if (not (os.path.isdir(self.script_dir))):
os.makedirs(self.script_dir)
self.qsub_invoke_log = "{0}.qil".format(self.get_job_file_name())
self.qsub_invoke_log_fullpath = "{0}/{1}".format(
self.script_dir, self.qsub_invoke_log)
self.configure()
def get_full_perm_code(permutation_info, scores_permutations_info, cspec):
master_permutation_info = {}
for key, val in permutation_info.items():
master_permutation_info[key] = val
if (len(scores_permutations_info) != 0):
for key, val in scores_permutations_info.items():
master_permutation_info[key] = val
full_perm_code = permutations.generate_perm_code(
master_permutation_info, cspec.concise_print_map,
permutations.INCLUDE_TRIALS)
return full_perm_code
def __init__(self, cspec, stdout):
self.cspec = cspec
self.job_number_for_perm_code_map = {}
self.perm_code_for_job_number_map = {}
self.cluster_script_for_perm_code_map = {}
self.perm_info_for_perm_code_map = {}
self.perm_code_for_perm_info_map = {}
self.result_dir_for_perm_code_map = {}
self.permuters_including_trials = cspec.get_permuters_trials_included()
self.permutation_info_list_full = permutations.expand_permutations(
self.permuters_including_trials)
self.run_perm_codes_list = []
self.job_num_width = self.get_job_number_width(
self.permutation_info_list_full)
# associate job numbers here and lookup from then on
user_job_number = 1
if cspec.first_job_number != '':
user_job_number = int(cspec.first_job_number)
for perm_info in self.permutation_info_list_full:
perm_code = permutations.generate_perm_code(
perm_info, cspec.concise_print_map,
permutations.INCLUDE_TRIALS)
self.run_perm_codes_list.append(perm_code)
user_job_number_as_string = get_formatted_user_job_number(
user_job_number, self.job_num_width)
self.job_number_for_perm_code_map[
perm_code] = user_job_number_as_string
self.perm_code_for_job_number_map[
user_job_number_as_string] = perm_code
self.perm_info_for_perm_code_map[perm_code] = perm_info
perm_info_string = "{0}".format(perm_info)
self.perm_code_for_perm_info_map[perm_info_string] = perm_code
resolved_results_dir = permutations.get_resolved_results_dir_for_permutation(
perm_info, cspec)
self.result_dir_for_perm_code_map[perm_code] = resolved_results_dir
cscript = cluster_script.ClusterScript(user_job_number_as_string,
perm_info, cspec,
perm_info['trial'], stdout)
self.cluster_script_for_perm_code_map[perm_code] = cscript
user_job_number = user_job_number + 1
def create_source_file_map(cspec):
logging.info('CREATING source file map')
source_file_map = {}
#need to add trials in with cspec.permuters before expanding
trials_list = cspec.get_trials_list()
permuters_with_trials = {}
for key, val in cspec.permuters.items():
permuters_with_trials[key] = val
permuters_with_trials['trial'] = trials_list
permutation_list = permutations.expand_permutations(permuters_with_trials)
for permutation_info in permutation_list:
pcode = permutations.generate_perm_code(permutation_info,
cspec.concise_print_map,
permutations.INCLUDE_TRIALS)
#print "permutation_code {0}".format(permutation_code)
permutation_output_dir = cspec.generate_results_dir_for_permutation(
pcode)
from_file_path_with_results_dir_resolved = "{0}/{1}".format(
permutation_output_dir, cspec.output_filename)
#from_file_path_with_results_dir_resolved = cspec.scores_from_filepath.replace('<permutation_output_dir>',permutation_output_dir)
#print "from_file_path_with_results_dir_resolved {0}".format(from_file_path_with_results_dir_resolved)
scores_permutations_list = permutations.expand_permutations(
cspec.scores_permuters)
if (len(scores_permutations_list) == 0):
fully_resolved_from_filepath = get_fully_resolved_from_filepath(
from_file_path_with_results_dir_resolved, permutation_info,
cspec, {})
full_perm_code = get_full_perm_code(permutation_info, {}, cspec)
source_file_map[full_perm_code] = fully_resolved_from_filepath
else:
for scores_permutations_info in scores_permutations_list:
fully_resolved_from_filepath = get_fully_resolved_from_filepath(
from_file_path_with_results_dir_resolved, permutation_info,
cspec, scores_permutations_info)
full_perm_code = get_full_perm_code(permutation_info,
scores_permutations_info,
cspec)
source_file_map[full_perm_code] = fully_resolved_from_filepath
#print "source_file_map[{0}] = {1}".format(full_perm_code, fully_resolved_from_filepath)
return source_file_map
def persist(self):
cspec = self.cspec
# generate the column names
ft = open(self.target_timings_path, 'w')
print "persisting {0}".format(self.target_timings_path)
# scores_y_axis:letter
# scores_x_axis:number,animal
x_permuters = {}
y_permuters = {}
for item in cspec.scores_x_axis:
x_permuters[item] = cspec.permuters[item]
for item in cspec.scores_y_axis:
y_permuters[item] = cspec.permuters[item]
x_permutations = permutations.expand_permutations(x_permuters)
y_permutations = permutations.expand_permutations(y_permuters)
# make a list of x permutation codes for use later
x_perm_codes = []
for x_permutation in x_permutations:
x_perm_codes.append(
permutations.generate_perm_code(x_permutation,
cspec.concise_print_map,
permutations.IGNORE_TRIALS))
medians = {}
# write the x_axis column names
header = "{0},".format(
pooled_results_file.beautify_header("{0}".format(
cspec.scores_y_axis)))
for x_permutation in x_permutations:
concise_x_permutation = permutations.generate_perm_code(
x_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
header = "{0}{1},".format(header, concise_x_permutation)
header = header.rstrip(',')
ft.write("{0}\n".format(header))
for y_permutation in y_permutations:
concise_y_permutation = permutations.generate_perm_code(
y_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
timings_line = "{0},".format(concise_y_permutation)
for x_permutation in x_permutations:
trials_list = cspec.get_trials_list()
trial_timing_values = []
for trial in trials_list:
cluster_job_perm_code = gen_cluster_job_perm_code_from_pieces(
y_permutation, x_permutation,
self.filename_permutation_info, cspec, trial)
timing_value = get_timing_value_for_run(
cluster_job_perm_code, self.cluster_runs, self.stdout)
#print 'timing_value : {0}'.format(timing_value)
trial_timing_values.append(timing_value)
median_timing = pooled_results_file.get_median(
trial_timing_values, True)
#print 'median_timing {0}'.format(median_timing)
timings_line = "{0}{1},".format(timings_line, median_timing)
x_perm_code = permutations.generate_perm_code(
x_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
pooled_results_file.record_median(x_perm_code, medians,
median_timing)
timings_line.rstrip(',')
ft.write("{0}\n".format(timings_line))
# add the averages line
line = "averages,"
for x_perm_code in x_perm_codes:
medians_list = medians[x_perm_code]
average = pooled_results_file.compute_average_medians(
medians_list, True)
line = "{0}{1},".format(line, average)
line = line.rstrip(',')
ft.write("{0}\n".format(line))
ft.close()
def get_job_number_string_for_permutation_info(self, perm_info):
pcode = permutations.generate_perm_code(perm_info,
self.cspec.concise_print_map,
permutations.INCLUDE_TRIALS)
result = self.job_number_for_perm_code_map[pcode]
return result
def persist(self):
cspec = self.cspec
# generate the column names
f = open(self.target_path, 'w')
self.stdout.println("persisting {0}".format(self.target_path))
# scores_y_axis:letter
# scores_x_axis:number,animal
x_permuters = {}
y_permuters = {}
for item in cspec.scores_x_axis:
x_permuters[item] = cspec.permuters[item]
for item in cspec.scores_y_axis:
y_permuters[item] = cspec.permuters[item]
x_permutations = permutations.expand_permutations(x_permuters)
y_permutations = permutations.expand_permutations(y_permuters)
# write the x_axis column names
header = "{0},".format(
beautify_header("{0}".format(cspec.scores_y_axis)))
for x_permutation in x_permutations:
concise_x_permutation = permutations.generate_perm_code(
x_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
header = "{0}{1},".format(header, concise_x_permutation)
header = header.rstrip(',')
f.write("{0}\n".format(header))
medians = {}
# make a list of x permutation codes for use later
x_perm_codes = []
for x_permutation in x_permutations:
x_perm_codes.append(
permutations.generate_perm_code(x_permutation,
cspec.concise_print_map,
permutations.IGNORE_TRIALS))
# the main loop
for y_permutation in y_permutations:
concise_y_permutation = permutations.generate_perm_code(
y_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
self.stdout.println('y axis: {0}'.format(concise_y_permutation))
line = "{0},".format(concise_y_permutation)
for x_permutation in x_permutations:
trials_list = cspec.get_trials_list()
trial_values = []
for trial in trials_list:
result_file_perm_code = gen_result_perm_code_from_pieces(
y_permutation, x_permutation,
self.filename_permutation_info, cspec, trial)
#print "self.source_file_map {0}".format(self.source_file_map)
source_file_path = self.source_file_map[
result_file_perm_code]
#print "SOURCE_FILE_PATH : {0}".format(source_file_path)
value = get_result_from_file(source_file_path,
cspec.scores_from_colname,
cspec.scores_from_rownum)
trial_values.append(value)
median_value = get_median(trial_values, False)
line = "{0}{1},".format(line, median_value)
x_perm_code = permutations.generate_perm_code(
x_permutation, cspec.concise_print_map,
permutations.IGNORE_TRIALS)
record_median(x_perm_code, medians, median_value)
line = line.rstrip(',')
f.write("{0}\n".format(line))
# add the averages line
line = "averages,"
for x_perm_code in x_perm_codes:
medians_list = medians[x_perm_code]
average = compute_average_medians(medians_list, False)
line = "{0}{1},".format(line, average)
line = line.rstrip(',')
f.write("{0}\n".format(line))
f.close()