def check_all(repo_kernel_file, threshold_list, top_k): total = 0 tp_list = [0]*len(threshold_list) fp_list = [0]*len(threshold_list) tn_list = [0]*len(threshold_list) fn_list = [0]*len(threshold_list) acc_list = [0]*len(threshold_list) # read kernel only once sim = Similarity() sim.read_graph_kernels(repo_kernel_file) with open(repo_kernel_file, 'r') as fi: for line in fi: line = line.rstrip() parts = line.split('\t') dot_file = parts[0] result_program_list_with_score = sim.find_top_k_similar_graphs(dot_file, 'g', top_k, 3) # num_iter = 3 path_parts = dot_file.split(os.sep) true_prob = path_parts[-4] total += 1 for (i, threshold) in enumerate(threshold_list): cr = check_result(true_prob, result_program_list_with_score, threshold) if cr=='tp': tp_list[i] += 1 elif cr=='fp': fp_list[i] += 1 elif cr=='fn': fn_list[i] += 1 else: tn_list[i] += 1 acc = check_top_k_result(true_prob, result_program_list_with_score, threshold, top_k) acc_list[i] += acc return total, tp_list, fp_list, tn_list, fn_list, acc_list
def find_top_k_similar_program(repo_kernel_file, user_prog_graph_dot_file, graph_name, k, num_iter, cluster_json): sim = Similarity() sim.read_graph_kernels(repo_kernel_file) result_program_list_with_score = sim.find_top_k_similar_graphs( user_prog_graph_dot_file, graph_name, k, num_iter, cluster_json) return result_program_list_with_score
def check_all(repo_kernel_file, threshold_list, top_k): total = 0 tp_list = [0] * len(threshold_list) fp_list = [0] * len(threshold_list) tn_list = [0] * len(threshold_list) fn_list = [0] * len(threshold_list) acc_list = [0] * len(threshold_list) # read kernel only once sim = Similarity() sim.read_graph_kernels(repo_kernel_file) with open(repo_kernel_file, 'r') as fi: for line in fi: line = line.rstrip() parts = line.split('\t') dot_file = parts[0] result_program_list_with_score = sim.find_top_k_similar_graphs( dot_file, 'g', top_k, 3) # num_iter = 3 path_parts = dot_file.split(os.sep) true_prob = path_parts[-4] total += 1 for (i, threshold) in enumerate(threshold_list): cr = check_result(true_prob, result_program_list_with_score, threshold) if cr == 'tp': tp_list[i] += 1 elif cr == 'fp': fp_list[i] += 1 elif cr == 'fn': fn_list[i] += 1 else: tn_list[i] += 1 acc = check_top_k_result(true_prob, result_program_list_with_score, threshold, top_k) acc_list[i] += acc return total, tp_list, fp_list, tn_list, fn_list, acc_list
def main(corpus, annotations, limit=3): """ SUMMARY: use case of the user-driven functionality of PASCALI. Scenario: User provides the concept of Sequence and the equivalent Java types, and the concept of sorted sequence and the relevant type invariant. Goal: learn how to get from Sequence -> Sorted Sequence. """ """ INPUT: annotations, dictionary mapping string -> list of strings OUTPUT: recompiles generic-inference-solver with new annotations""" run_pa2checker(annotations) """ Look for new mapping from 'ontology concepts'->'java type' and run checker framework. Should be implemented in type_inference Mapping example: Sequence -> java.lang.Array, java.util.List, LinkedHashSet, etc. INPUT: corpus, file containing set of concept->java_type mapping OUTPUT: Set of jaif files that are merged into the classes. The jaif files are stored as default.jaif in each project's directory. BODY: This also triggers back-end labeled graph generation. """ for project in corpus: run_inference(project) """ Missing step: interact with PA to add a definition of Sorted Sequence which is a specialization of Sequence that has a sortedness invariants. The sortedness invariant gets turned into a Daikon template INPUT: user interaction OUTPUT: type_annotation and type_invariant (for sorted sequence) """ ordering_operator = "<=" ontology_invariant_file = "TODO_from_Howie.txt" with open(ontology_invariant_file, 'w') as f: f.write(ordering_operator) invariant_name = "TODO_sorted_sequence" daikon_pattern_java_file = ontology_to_daikon.create_daikon_invariant( ontology_invariant_file, invariant_name) """ Find all methods that have one input parameter annotated as Sequence and return a variable also annotated as Sequence. INPUT: The corpus and the desired annotations on the method signature OUTPUT: List of methods that have the desired signature. NOTE: This is a stub and will be implemented as LB query in the future. """ sig_methods = find_methods_with_signature(corpus, "@ontology.qual.Sequence", ["@ontology.qual.Sequence"]) print("\n ************") print( "The following corpus methods have the signature Sequence->Sequence {}:" ) for (project, package, clazz, method) in sig_methods: print("{}:\t{}.{}.{}".format(project, package, clazz, method)) print("\n ************") """ Search for methods that have a return type annotated with Sequence and for which we can establish a sortedness invariant (may done by LB). INPUT: dtrace file of project daikon_pattern_java_file that we want to check on the dtrace file. OUTPUT: list of ppt names that establish the invariant. Here a ppt is a Daikon program point, s.a. test01.TestClass01.sort(int[]):::EXIT Note: this step translate the type_invariant into a Daikon template (which is a Java file). """ pattern_class_name = invariant_name pattern_class_dir = os.path.join(common.WORKING_DIR, "invClass") if os.path.isdir(pattern_class_dir): shutil.rmtree(pattern_class_dir) os.mkdir(pattern_class_dir) cmd = [ "javac", "-g", "-classpath", common.get_jar('daikon.jar'), daikon_pattern_java_file, "-d", pattern_class_dir ] common.run_cmd(cmd) list_of_methods = [] for project in corpus: dtrace_file = backend.get_dtrace_file_for_project(project) if not dtrace_file: print("Ignoring folder {} because it does not contain dtrace file". format(project)) continue ppt_names = inv_check.find_ppts_that_establish_inv( dtrace_file, pattern_class_dir, pattern_class_name) methods = set() for ppt in ppt_names: method_name = ppt[:ppt.find(':::EXIT')] methods.add(method_name) list_of_methods += [(project, methods)] print("\n ************") print( "The following corpus methods return a sequence sorted by {}:".format( ordering_operator)) for project, methods in list_of_methods: if len(methods) > 0: print(project) for m in methods: print("\t{}".format(m)) print("\n ************") shutil.rmtree(pattern_class_dir) """ Expansion of dynamic analysis results .... Find a list of similar methods that are similar to the ones found above (list_of_methods). INPUT: list_of_methods, corpus with labeled graphs generated, threshold value for similarity, OUTPUT: superset_list_of_methods """ # WENCHAO print( "Expanding the dynamic analysis results using graph-based similarity:") union_set = set() for project, methods in list_of_methods: # map Daikon output on sort method to method signature in methods.txt in generated graphs for m in methods: method_name = common.get_method_from_daikon_out(m) #kernel_file = common.get_kernel_path(project) method_file = common.get_method_path(project) dot_name = common.find_dot_name(method_name, method_file) if dot_name: # find the right dot file for each method dot_file = common.get_dot_path(project, dot_name) # find all graphs that are similar to it using WL based on some threshold sys.path.append(os.path.join(common.WORKING_DIR, 'simprog')) from similarity import Similarity sim = Similarity() sim.read_graph_kernels( os.path.join(common.WORKING_DIR, "corpus_kernel.txt")) top_k = 3 iter_num = 3 result_program_list_with_score = sim.find_top_k_similar_graphs( dot_file, 'g', top_k, iter_num) print(project + ":") print(result_program_list_with_score) result_set = set( [x[0] for x in result_program_list_with_score]) # take the union of all these graphs union_set = union_set | result_set print("Expanded set:") print([x.split('/')[-4] for x in union_set]) # return this set as a list of (project, method) fo = open("methods.txt", "w") expanded_list = [] for dot_path in union_set: method_summary = common.get_method_summary_from_dot_path(dot_path) fo.write(method_summary) fo.write("\n") fo.close() """ Update the type annotations for the expanded dynamic analysis results. INPUT: superset_list_of_methods, annotation to be added OUTPUT: nothing EFFECT: updates the type annotations of the methods in superset_list_of_methods. This requires some additional checks to make sure that the methods actually perform some kind of sorting. Note that we do it on the superset because the original list_of_methods might miss many implementations because fuzz testing could not reach them. """ for class_file in []: # MARTIN generated_jaif_file = "TODO" insert_jaif.merge_jaif_into_class(class_file, generated_jaif_file) """ Ordering of expanded dynamic analysis results .... Find the k 'best' implementations in superset of list_of_methods INPUT: superset_list_of_methods, corpus, k OUTPUT: k_list_of_methods Note: similarity score is used. may consider using other scores; e.g., TODO:??? """ #TODO: create input file for huascar where each line is formatted like: # ../corpus/Sort05/src/Sort05.java::sort(int[]):int[] ordering_dir = os.path.join(common.WORKING_DIR, "ordering_results/") methods_file = os.path.join(common.WORKING_DIR, 'methods.txt') with common.cd(ordering_dir): #TODO generate a proper relevant methods file. cmd = [ "./run.sh", "-k", "{}".format(limit), "-t", "typicality", "-f", methods_file ] common.run_cmd(cmd, print_output=True) """
def main(corpus, annotations): """ SUMMARY: use case of the user-driven functionality of PASCALI. Scenario: User provides the concept of Sequence and the equivalent Java types, and the concept of sorted sequence and the relevant type invariant. Goal: learn how to get from Sequence -> Sorted Sequence. """ """ INPUT: annotations, dictionary mapping string -> list of strings OUTPUT: recompiles generic-inference-solver with new annotations""" run_pa2checker(annotations) """ Look for new mapping from 'ontology concepts'->'java type' and run checker framework. Should be implemented in type_inference Mapping example: Sequence -> java.lang.Array, java.util.List, LinkedHashSet, etc. INPUT: corpus, file containing set of concept->java_type mapping OUTPUT: Set of jaif files that are merged into the classes. The jaif files are stored as default.jaif in each project's directory. BODY: This also triggers back-end labeled graph generation. """ for project in corpus: run_inference(project) """ Missing step: interact with PA to add a definition of Sorted Sequence which is a specialization of Sequence that has a sortedness invariants. The sortedness invariant gets turned into a Daikon template INPUT: user interaction OUTPUT: type_annotation and type_invariant (for sorted sequence) """ ordering_operator = "<=" ontology_invariant_file = "TODO_from_Howie.txt" with open(ontology_invariant_file, 'w') as f: f.write(ordering_operator) invariant_name = "TODO_sorted_sequence" daikon_pattern_java_file = ontology_to_daikon.create_daikon_invariant(ontology_invariant_file, invariant_name) """ Find all methods that have one input parameter annotated as Sequence and return a variable also annotated as Sequence. INPUT: The corpus and the desired annotations on the method signature OUTPUT: List of methods that have the desired signature. NOTE: This is a stub and will be implemented as LB query in the future. """ sig_methods = find_methods_with_signature(corpus, "@ontology.qual.Sequence", ["@ontology.qual.Sequence"]) print ("\n ************") print ("The following corpus methods have the signature Sequence->Sequence {}:") for (project, package, clazz, method) in sig_methods: print("{}:\t{}.{}.{}".format(project, package, clazz, method)) print ("\n ************") """ Search for methods that have a return type annotated with Sequence and for which we can establish a sortedness invariant (may done by LB). INPUT: dtrace file of project daikon_pattern_java_file that we want to check on the dtrace file. OUTPUT: list of ppt names that establish the invariant. Here a ppt is a Daikon program point, s.a. test01.TestClass01.sort(int[]):::EXIT Note: this step translate the type_invariant into a Daikon template (which is a Java file). """ pattern_class_name = invariant_name pattern_class_dir = os.path.join(common.WORKING_DIR, "invClass") if os.path.isdir(pattern_class_dir): shutil.rmtree(pattern_class_dir) os.mkdir(pattern_class_dir) cmd = ["javac", "-g", "-classpath", common.get_jar('daikon.jar'), daikon_pattern_java_file, "-d", pattern_class_dir] common.run_cmd(cmd) list_of_methods = [] for project in corpus: dtrace_file = backend.get_dtrace_file_for_project(project) if not dtrace_file: print ("Ignoring folder {} because it does not contain dtrace file".format(project)) continue ppt_names = inv_check.find_ppts_that_establish_inv(dtrace_file, pattern_class_dir, pattern_class_name) methods = set() for ppt in ppt_names: method_name = ppt[:ppt.find(':::EXIT')] methods.add(method_name) list_of_methods +=[(project, methods)] print ("\n ************") print ("The following corpus methods return a sequence sorted by {}:".format(ordering_operator)) for project, methods in list_of_methods: if len(methods)>0: print (project) for m in methods: print("\t{}".format(m)) print ("\n ************") shutil.rmtree(pattern_class_dir) """ Expansion of dynamic analysis results .... Find a list of similar methods that are similar to the ones found above (list_of_methods). INPUT: list_of_methods, corpus with labeled graphs generated, threshold value for similarity, OUTPUT: superset_list_of_methods """ # WENCHAO print("Expanding the dynamic analysis results using graph-based similarity:") union_set = set() for project, methods in list_of_methods: # map Daikon output on sort method to method signature in methods.txt in generated graphs for m in methods: method_name = common.get_method_from_daikon_out(m) #kernel_file = common.get_kernel_path(project) method_file = common.get_method_path(project) dot_name = common.find_dot_name(method_name, method_file) if dot_name: # find the right dot file for each method dot_file = common.get_dot_path(project, dot_name) # find all graphs that are similar to it using WL based on some threshold sys.path.insert(0, 'simprog') from similarity import Similarity sim = Similarity() sim.read_graph_kernels("corpus_kernel.txt") top_k = 3 iter_num = 3 result_program_list_with_score = sim.find_top_k_similar_graphs(dot_file, 'g', top_k, iter_num) print(project+":") print(result_program_list_with_score) result_set = set([x[0] for x in result_program_list_with_score]) # take the union of all these graphs union_set = union_set | result_set print("Expanded set:") print([x.split('/')[-4] for x in union_set]) # return this set as a list of (project, method) fo = open("methods.txt", "w") expanded_list = [] for dot_path in union_set: method_summary = common.get_method_summary_from_dot_path(dot_path) fo.write(method_summary) fo.write("\n") fo.close() """ Update the type annotations for the expanded dynamic analysis results. INPUT: superset_list_of_methods, annotation to be added OUTPUT: nothing EFFECT: updates the type annotations of the methods in superset_list_of_methods. This requires some additional checks to make sure that the methods actually perform some kind of sorting. Note that we do it on the superset because the original list_of_methods might miss many implementations because fuzz testing could not reach them. """ for class_file in []: # MARTIN generated_jaif_file = "TODO" insert_jaif.merge_jaif_into_class(class_file, generated_jaif_file) """ Ordering of expanded dynamic analysis results .... Find the k 'best' implementations in superset of list_of_methods INPUT: superset_list_of_methods, corpus, k OUTPUT: k_list_of_methods Note: similarity score is used. may consider using other scores; e.g., TODO:??? """ #TODO: create input file for huascar where each line is formatted like: # ../corpus/Sort05/src/Sort05.java::sort(int[]):int[] ordering_dir = os.path.join(common.WORKING_DIR, "ordering_results/") methods_file = os.path.join(common.WORKING_DIR, 'methods.txt') with common.cd(ordering_dir): #TODO generate a proper relevant methods file. cmd = ["./run.sh", "-k", "3", "-t", "typicality", "-f", methods_file] common.run_cmd(cmd, print_output=True) """ Close the loop and add the best implementation found in the previous step back to the ontology. INPUT: k_list_of_methods OUTPUT: patch file for the ontology. Worst case: just add the 'best' implementation found in the corpus as a blob to the ontology. Best case: generate an equivalent flow-graph in the ontology. """ print "TODO" # ALL
def find_top_k_similar_program(repo_kernel_file, user_prog_graph_dot_file, graph_name, k, num_iter): sim = Similarity() sim.read_graph_kernels(repo_kernel_file) result_program_list_with_score = sim.find_top_k_similar_graphs(user_prog_graph_dot_file, graph_name, k, num_iter) return result_program_list_with_score