def simplified_pairs_unordered():
if len(sys.argv) > 1:
destination = sys.argv[1]
else:
print("please provide a destination file in the format <filename>.csv")
exit(1)
destfile = open(destination, mode="w")
destwriter = csv.writer(destfile)
#a list of lists; a master list containing lists of programs broken down by problem they solve
organized_programs = simp_success_only()
all_progs = organized_programs[0]
pairs = {}
freqs = {}
#this includes doubles in the program, should fix
for program in all_progs:
program = simplify(program)
for i in range(0, len(program) - 1):
func = program[i]
other = program[i + 1]
is_constant = check_if_constant(func)
if not is_constant:
freqs = add_freqs(func, freqs)
if not check_if_constant(other):
if (func, other) in pairs:
pairs[(func, other)] += 1
elif (other, func) in pairs:
pairs[(other, func)] += 1
else:
pairs[(func, other)] = 1
if i == len(program) - 2:
if not check_if_constant(program[i + 1]):
add_freqs(other, freqs)
all_pairs = []
all_freqs = []
total_freqs = []
for (f1, f2) in pairs:
significant = significance(f1, f2, freqs, pairs)
if significant:
all_pairs.append((f1, f2))
all_freqs.append(pairs[(f1, f2)])
total_freqs.append((freqs[f1], freqs[f2]))
print "%s, %s: %i" % (f1, f2, pairs[(f1, f2)])
print "%s: %i" % (f1, freqs[f1])
print "%s: %i" % (f2, freqs[f2])
print
destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_freqs)
return newprog
if len(sys.argv) > 1:
destination = sys.argv[1]
else:
print("please provide a destination file in the format <filename>.csv")
exit(1)
destfile = open(destination, mode="w")
destwriter = csv.writer(destfile)
organized_programs = simp_success_only()
all_programs = organized_programs[0]
number_io = organized_programs[1]
checksum = organized_programs[2]
collatz_numbers = organized_programs[3]
compare_string_lengths = organized_programs[4]
count_odds = organized_programs[5]
digits = organized_programs[6]
double_letters = organized_programs[7]
even_squares = organized_programs[8]
for_loop_index = organized_programs[9]
grade = organized_programs[10]
last_index_of_zero = organized_programs[11]
median = organized_programs[12]
mirror_image = organized_programs[13]
def simplified_pairs():
if len(sys.argv) > 1:
destination = sys.argv[1]
else:
print("please provide a destination file in the format <filename>.csv")
exit(1)
destfile = open(destination, mode="w")
destwriter = csv.writer(destfile)
#final = [all_programs, number_io, checksum, collatz_numbers, compare_string_lengths, count_odds, digits, double_letters, even_squares, for_loop_index, grade, last_index_of_zero, median, mirror_image, negative_to_zero, pig_latin, replace_space_with_newline, scrabble_score, small_or_large, smallest, string_differences, string_lengths_backwards, sum_of_squares, super_anagrams, syllables, vector_average, vectors_summed, wallis_pi, word_stats, x_word_lines]
IO = [{}, {}]
arithmetic = [{}, {}]
comparison = [{}, {}]
boolean = [{}, {}]
string_handling = [{}, {}]
vectors = [{}, {}]
#a list of lists; a master list containing lists of programs broken down by problem they solve
#just change this line for unsimplified
organized_programs = simp_success_only()
#all_progs = organized_programs[0]
for prob in range (1, len(organized_programs)):
if prob == 1: #*************************************************************
tags = [IO, arithmetic]
if prob == 2:
tags = [string_handling, arithmetic, IO]
if prob == 3:
tags = [arithmetic, comparison]
if prob == 4: #*************************************************************
tags = [boolean, comparison, string_handling]
if prob == 5:
tags = [vectors, arithmetic]
if prob == 6:
tags = [IO, arithmetic, string_handling]
if prob == 7: #*************************************************************
tags = [string_handling, IO, comparison]
if prob == 8:
tags = [IO, arithmetic]
if prob == 9:
tags = [IO, arithmetic]
if prob == 10:
tags = [IO, comparison]
if prob == 11: #*************************************************************
tags = [vectors, comparison]
if prob == 12: #*************************************************************
tags = [IO, comparison]
if prob == 13: #*************************************************************
tags = [vectors, boolean, comparison]
if prob == 14: #*************************************************************
tags = [vectors, comparison]
if prob == 15:
tags = [string_handling, IO]
if prob == 16: #*************************************************************
tags = [string_handling, IO, arithmetic]
if prob == 17: #*************************************************************
tags = [string_handling, arithmetic]
if prob == 18:
tags = [IO, comparison]
if prob == 19:
tags = [IO, comparison]
if prob == 20:
tags = [string_handling, comparison, IO]
if prob == 21:
tags = [IO, string_handling, vectors]
if prob == 22:
tags = [arithmetic]
if prob == 23:
tags = [boolean, string_handling, comparison]
if prob == 24: #*************************************************************
tags = [IO, string_handling, arithmetic]
if prob == 25: #*************************************************************
tags = [vectors, arithmetic]
if prob == 26:
tags = [vectors, arithmetic]
if prob == 27:
tags = [arithmetic]
if prob == 28:
tags = [IO, string_handling, arithmetic]
if prob == 29: #*************************************************************
tags = [string_handling, IO]
problem = organized_programs[prob]
(dists, counts) = calculate_distances(problem)
freqs = {}
pairs = {}
for oprogram in problem:
program = simplify(oprogram)
for i in range(0, len(program) - 1):
func = program[i]
rest = program[i+1:]
is_constant = check_if_constant(func)
if not is_constant:
freqs = add_freqs(func, freqs)
for other in rest:
if not check_if_constant(other):
if (func, other) in pairs:
pairs[(func, other)] += 1
#comment this line for ordered pairs, uncomment for unordered pairs
elif (other, func) in pairs:
pairs[(other, func)] += 1
else:
pairs[(func, other)] = 1
if i == len(program) - 2:
if not check_if_constant(program[i+1]):
freqs = add_freqs(program[i+1], freqs)
for (f1, f2) in pairs:
pair = (f1, f2)
if pair in dists:
avgdist = sum(dists[(f1, f2)]) / float(counts[(f1, f2)])
else:
avgdist = sum(dists[(f2, f1)]) / float(counts[(f2, f1)])
frequency = float(pairs[pair]) / len(problem)
for tag in tags:
#each tag is TAG = [pairs, dists], where pairs and dists are dictionaries containing lists
if (f1, f2) in tag[0]:
tag[0][(f1, f2)].append(frequency)
tag[1][(f1, f2)].append(avgdist)
elif (f2, f1) in tag[0]:
tag[0][(f2, f1)].append(frequency)
tag[1][(f2, f1)].append(avgdist)
else:
tag[0][(f1, f2)] = [frequency]
tag[1][(f1, f2)] = [avgdist]
"""
significant = significance(f1, f2, freqs, pairs)
if significant:
print f1, f2
if (f1, f2) in dists:
avgdist = sum(dists[(f1, f2)]) / float(counts[(f1, f2)])
else:
avgdist = sum(dists[(f2, f1)]) / float(counts[(f2, f1)])
#print avgdist
print
all_pairs.append((f1, f2))
all_freqs.append(pairs[(f1, f2)])
total_freqs.append((freqs[f1], freqs[f2]))
total_dists.append(avgdist)"""
pairs[(f1, f2)] = 0
for elem in freqs:
freqs[elem] = 0
"""destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_freqs)
destwriter.writerow(total_dists)"""
all_pairs = []
all_freqs = []
total_dists = []
destwriter.writerow(["IO"])
for combo in IO[0]:
#IO[0][combo] = list of frequencies
avg_frequency = sum(IO[0][combo]) / 5
#IO[1][combo] = list of average distances
avg_distance = sum(IO[1][combo]) / len(IO[1][combo])
#print avg_distance
if avg_frequency >= .3:
all_pairs.append(combo)
print combo
print IO[0][combo]
#print len(IO)
print avg_frequency
all_freqs.append(avg_frequency)
total_dists.append(avg_distance)
#print avg_distance
print
destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_dists)
all_pairs = []
all_freqs = []
total_dists = []
destwriter.writerow(["arithmetic"])
for combo in arithmetic[0]:
#arithmetic[0][combo] = list of frequencies
avg_frequency = sum(arithmetic[0][combo]) / 5
#arithmetic[1][combo] = list of average distances
avg_distance = sum(arithmetic[1][combo]) / len(arithmetic[1][combo])
#print avg_distance
if avg_frequency >= .3:
all_pairs.append(combo)
#print combo
#print avg_frequency
all_freqs.append(avg_frequency)
total_dists.append(avg_distance)
#print avg_distance
#print
destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_dists)
all_pairs = []
all_freqs = []
total_dists = []
destwriter.writerow(["comparison"])
for combo in comparison[0]:
#comparison[0][combo] = list of frequencies
avg_frequency = sum(comparison[0][combo]) / 6
#comparison[1][combo] = list of average distances
avg_distance = sum(comparison[1][combo]) / len(comparison[1][combo])
#print avg_distance
if avg_frequency >= .3:
all_pairs.append(combo)
#print combo
#print avg_frequency
all_freqs.append(avg_frequency)
total_dists.append(avg_distance)
#print avg_distance
#print
destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_dists)
all_pairs = []
all_freqs = []
total_dists = []
destwriter.writerow(["boolean"])
for combo in boolean[0]:
#boolean[0][combo] = list of frequencies
avg_frequency = sum(boolean[0][combo]) / 2
#boolean[1][combo] = list of average distances
avg_distance = sum(boolean[1][combo]) / len(boolean[1][combo])
#print avg_distance
if avg_frequency >= .3:
all_pairs.append(combo)
#print combo
#print avg_frequency
all_freqs.append(avg_frequency)
total_dists.append(avg_distance)
#print avg_distance
#print
destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_dists)
all_pairs = []
all_freqs = []
total_dists = []
destwriter.writerow(["string_handling"])
for combo in string_handling[0]:
#string_handling[0][combo] = list of frequencies
avg_frequency = sum(string_handling[0][combo]) / 6
#string_handling[1][combo] = list of average distances
avg_distance = sum(string_handling[1][combo]) / len(string_handling[1][combo])
#print avg_distance
if avg_frequency >= .3:
all_pairs.append(combo)
#print combo
#print avg_frequency
all_freqs.append(avg_frequency)
total_dists.append(avg_distance)
#print avg_distance
#print
destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_dists)
all_pairs = []
all_freqs = []
total_dists = []
destwriter.writerow(["vectors"])
for combo in vectors[0]:
#vectors[0][combo] = list of frequencies
avg_frequency = sum(vectors[0][combo]) / 4
#vectors[1][combo] = list of average distances
avg_distance = sum(vectors[1][combo]) / len(vectors[1][combo])
#print avg_distance
if avg_frequency >= .3:
all_pairs.append(combo)
#print combo
#print avg_frequency
all_freqs.append(avg_frequency)
total_dists.append(avg_distance)
#print avg_distance
#print
destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_dists)
def simplified_pairs():
if len(sys.argv) > 1:
destination = sys.argv[1]
else:
print("please provide a destination file in the format <filename>.csv")
exit(1)
destfile = open(destination, mode="w")
destwriter = csv.writer(destfile)
#a list of lists; a master list containing lists of programs broken down by problem they solve
#just change this line for unsimplified
organized_programs = simp_success_only()
all_progs = organized_programs[0]
pairs = {}
freqs = {}
(dists, counts) = calculate_distances(all_progs)
for oprogram in all_progs:
program = simplify(oprogram)
for i in range(0, len(program) - 1):
func = program[i]
rest = program[i+1:]
is_constant = check_if_constant(func)
if not is_constant:
freqs = add_freqs(func, freqs)
for other in rest:
if not check_if_constant(other):
if (func, other) in pairs:
pairs[(func, other)] += 1
#comment this line for ordered pairs, uncomment for unordered pairs
#elif (other, func) in pairs:
# pairs[(other, func)] += 1
else:
pairs[(func, other)] = 1
if i == len(program) - 2:
if not check_if_constant(program[i+1]):
add_freqs(program[i+1], freqs)
all_pairs = []
all_freqs = []
total_freqs = []
total_dists = []
minmax_dists = []
for (f1, f2) in pairs:
significant = significance(f1, f2, freqs, pairs)
if significant:
print f1, f2
if (f1, f2) in dists:
avgdist = sum(dists[(f1, f2)]) / float(counts[(f1, f2)])
distlst = dists[(f1, f2)]
#distlst.sort()
print distlst
minimum = min(dists[(f1, f2)])
maximum = max(dists[(f1, f2)])
else:
avgdist = sum(dists[(f2, f1)]) / float(counts[(f2, f1)])
distlst = dists[(f2, f1)]
#distlst.sort()
print distlst
minimum = min(dists[(f2, f1)])
maximum = max(dists[(f2, f1)])
print avgdist
print
all_pairs.append((f1, f2))
all_freqs.append(pairs[(f1, f2)])
total_freqs.append((freqs[f1], freqs[f2]))
total_dists.append(avgdist)
minmax_dists.append((minimum, maximum))
destwriter.writerow(all_pairs)
destwriter.writerow(all_freqs)
destwriter.writerow(total_freqs)
destwriter.writerow(total_dists)
destwriter.writerow(minmax_dists)
def goodfuncs1():
"""Calculates the frequency of each function that is returned by get_funcs_list for each tag, then
writes that data to a CSV
Frequency is calculated as number of programs that contain that function, not number of occurances of that function"""
tags = {"I/O" : ["number-io", "checksum", "digits", "double-letters", "even-squares", "for-loop-index", "grade", "median", "negative-to-zero", "pig-latin", "replace-space-with-newline", "small-or-large", "smallest", "string-differences", "string-lengths-backwards", "syllables", "word-stats", "x-word-lines"],
"arithmetic" : ["number-io", "checksum", "collatz-numbers", "count-odds", "digits", "even-squares", "for-loop-index", "replace-space-with-newline", "scrabble-score", "sum-of-squares", "syllables", "vector-average", "vectors-summed", "wallis-pi", "word-stats"],
"comparison" : ["collatz-numbers", "compare-string-lengths", "double-letters", "grade", "last-index-of-zero", "median", "mirror-image", "negative-to-zero", "small-or-large", "smallest","string-differences", "super-anagrams"],
"boolean" : ["compare-string-lengths", "mirror-image", "super-anagrams"],
"string_handling" : ["checksum", "compare-string-lengths", "digits", "double-letters", "pig-latin", "replace-space-with-newline", "scrabble-score", "string-differences", "string-lengths-backwards", "super-anagrams", "syllables", "word-stats", "x-word-lines"],
"vectors" : ["count-odds", "last-index-of-zero", "mirror-image", "negative-to-zero", "string-lengths-backwards", "vector-average", "vectors-summed"]}
IO = []
arithmetic = []
comparison = []
boolean = []
string_handling = []
vectors = []
all_programs = []
#l stands for location
IOl = []
arithmeticl = []
comparisonl = []
booleanl = []
string_handlingl = []
vectorsl = []
all_programsl = []
#keeps track of how many problems are in each category so that the averages can be calculated
iocount = 0
arithcount = 0
compcount = 0
boolcount = 0
strcount = 0
vectorcount = 0
#ensures the user provides a csv file to write to
if len(sys.argv) > 1:
destination = sys.argv[1]
else:
print("please provide a destination file in the format <filename>.csv")
exit(1)
destfile = open(destination, mode="w")
destwriter = csv.writer(destfile)
#a list of lists; a master list containing lists of programs broken down by problem they solve
#organized_programs = success_only()
organized_programs = simp_success_only()
#I removed file from this list because no program with that tag has succeeded, providing no data
problems = ["all_programs", "number-io", "checksum", "collatz-numbers", "compare-string-lengths", "count-odds", "digits", "double-letters", "even-squares", "for-loop-index", "grade", "last-index-of-zero", "median", "mirror-image", "negative-to-zero", "pig-latin", "replace-space-with-newline", "scrabble-score", "small-or-large", "smallest", "string-differences", "string-lengths-backwards", "sum-of-squares", "super-anagrams", "syllables", "vector-average", "vectors-summed", "wallis-pi", "word-stats", "x-word-lines"]
#all functions that had a frequency of .6 or higher for at least one problem
all_funcs = get_funcs_list(organized_programs)
#gets the data for each of the above functions for each problem
for prob in problems:
if prob == "all_programs":
programs = organized_programs[0]
if prob == "number-io":
programs = organized_programs[1]
if prob == "checksum":
programs = organized_programs[2]
if prob == "collatz-numbers":
programs = organized_programs[3]
if prob == "compare-string-lengths":
programs = organized_programs[4]
if prob == "count-odds":
programs = organized_programs[5]
if prob == "digits":
programs = organized_programs[6]
if prob == "double-letters":
programs = organized_programs[7]
if prob == "even-squares":
programs = organized_programs[8]
if prob == "for-loop-index":
programs = organized_programs[9]
if prob == "grade":
programs = organized_programs[10]
if prob == "last-index-of-zero":
programs = organized_programs[11]
if prob == "median":
programs = organized_programs[12]
if prob == "mirror-image":
programs = organized_programs[13]
if prob == "negative-to-zero":
programs = organized_programs[14]
if prob == "pig-latin":
programs = organized_programs[15]
if prob == "replace-space-with-newline":
programs = organized_programs[16]
if prob == "scrabble-score":
programs = organized_programs[17]
if prob == "small-or-large":
programs = organized_programs[18]
if prob == "smallest":
programs = organized_programs[19]
if prob == "string-differences":
programs = organized_programs[20]
if prob == "string-lengths-backwards":
programs = organized_programs[21]
if prob == "sum-of-squares":
programs = organized_programs[22]
if prob == "super-anagrams":
programs = organized_programs[23]
if prob == "syllables":
programs = organized_programs[24]
if prob == "vector-average":
programs = organized_programs[25]
if prob == "vectors-summed":
programs = organized_programs[26]
if prob == "wallis-pi":
programs = organized_programs[27]
if prob == "word-stats":
programs = organized_programs[28]
if prob == "x-word-lines":
programs = organized_programs[29]
#maximum number of occurances is equal to the number of programs
max_frequency = len(programs)
freqs = {}
locations = {}
#calculates the number of occurances
for program in programs:
program = fix_strings(program) #7/16 ******************I RECENTLY CHANGED THIS SECTION SO PLEASE DOUBLE CHECK IT WOOLSOCK
#this length includes constants
prog_len = len(program)
this_run = []
#print_prog(program)
for x in range(0, len(program)):
func = program[x]
#ACCOUNTS FOR CONSTANTS
is_constant = check_if_constant(func)
if (not is_constant) and not (func in this_run):
this_run.append(func)
if func in freqs:
freqs[func] += 1
locations[func].append(float(x) / prog_len)
else:
freqs[func] = 1
#x = location of function in program
locations[func] = [float(x) / prog_len]
#only does the rest if there are any solution programs for the problem in question
if max_frequency > 0:
print prob
print "Max possible occurances: %i" % max_frequency
print
#iterates over all functions in all_funcs
for i in range(0, len(all_funcs)):
function = all_funcs[i]
#if this function doesn't occur in this problem, we must set frequency to zero
# otherwise, it is the number of times the function occurs over the total number of programs
if function in freqs:
frequency = (freqs[function] / float(max_frequency))
#print freqs[function]
#print locations[function]
loc = sum(locations[function]) / len(locations[function])
#print func_loc
#print
else:
frequency = 0.0
if prob == "all_programs":
all_programs.append(frequency)
#if this is the first IO problem, the the list will be empty so we simply append
#if it isn't, then the list already contains an entry for each func so we add to each entry
if prob in tags["I/O"]:
if len(IO) - 1 < i:
IO.append(frequency)
IOl.append(loc)
else:
IO[i] = IO[i] + frequency
IOl[i] = IOl[i] + loc
if i == len(all_funcs) - 1:
iocount += 1
if prob in tags["arithmetic"]:
if len(arithmetic) - 1 < i:
arithmetic.append(frequency)
arithmeticl.append(loc)
else:
arithmetic[i] = arithmetic[i] + frequency
arithmeticl[i] = arithmeticl[i] + loc
if i == len(all_funcs) - 1:
arithcount += 1
if prob in tags["comparison"]:
if len(comparison) - 1 < i:
comparison.append(frequency)
comparisonl.append(loc)
else:
comparison[i] = comparison[i] + frequency
comparisonl[i] = comparisonl[i] + loc
if i == len(all_funcs) - 1:
compcount += 1
if prob in tags["boolean"]:
if len(boolean) - 1 < i:
boolean.append(frequency)
booleanl.append(loc)
else:
boolean[i] = boolean[i] + frequency
booleanl[i] = booleanl[i] + loc
if i == len(all_funcs) - 1:
boolcount += 1
if prob in tags["string_handling"]:
if len(string_handling) - 1 < i:
string_handling.append(frequency)
string_handlingl.append(loc)
else:
string_handling[i] = string_handling[i] + frequency
string_handlingl[i] = string_handlingl[i] + loc
if i == len(all_funcs) - 1:
strcount += 1
if prob in tags["vectors"]:
if len(vectors) - 1 < i:
vectors.append(frequency)
vectorsl.append(loc)
else:
vectors[i] = vectors[i] + frequency
vectorsl[i] = vectorsl[i] + loc
if i == len(all_funcs) - 1:
vectorcount += 1
#goes through each list and calculates the average frequency
for a in range(0, len(all_funcs)):
element = IO[a]
IO[a] = element / iocount
IOl[a] = IOl[a] / iocount
element = arithmetic[a]
arithmetic[a] = element / arithcount
arithmeticl[a] = arithmeticl[a] / arithcount
element = comparison[a]
comparison[a] = element / compcount
comparisonl[a] = comparisonl[a] / compcount
element = boolean[a]
boolean[a] = element / boolcount
booleanl[a] = booleanl[a] / boolcount
element = string_handling[a]
string_handling[a] = element / strcount
string_handlingl[a] = string_handlingl[a] / strcount
element = vectors[a]
vectors[a] = element / vectorcount
vectorsl[a] = vectorsl[a] / vectorcount
destwriter.writerow(["Tag"] + all_funcs)
destwriter.writerow(["IO"] + IO)
destwriter.writerow(["Locations"] + IOl)
destwriter.writerow(["Arithmetic"] + arithmetic)
destwriter.writerow(["Locations"] + arithmeticl)
destwriter.writerow(["Comparison"] + comparison)
destwriter.writerow(["Locations"] + comparisonl)
destwriter.writerow(["Boolean"] + boolean)
destwriter.writerow(["Locations"] + booleanl)
destwriter.writerow(["String Handling"] + string_handling)
destwriter.writerow(["Locations"] + string_handlingl)
destwriter.writerow(["Vectors"] + vectors)
destwriter.writerow(["Locations"] + vectorsl)
destwriter.writerow(["All Programs"] + all_programs)