def main(): parser = Parser() parser.add_function( find_all_matches("cost:all", r"Plan cost: (.+)\n", type=float)) parser.add_function( find_all_matches("steps:all", r"Plan length: (.+) step\(s\).\n", type=float)) parser.add_function(reduce_to_min("cost:all", "cost")) parser.add_function(reduce_to_min("steps:all", "steps")) parser.add_function(coverage) parser.parse()
def main(): print("Running singularity parser") parser = Parser() parser.add_pattern( "planner_exit_code", r"run-planner exit code: (.+)\n", type=int, file="driver.log", required=True, ) parser.add_pattern("node", r"node: (.+)\n", type=str, file="driver.log", required=True) parser.add_pattern( "planner_wall_clock_time", r"run-planner wall-clock time: (.+)s", type=float, file="driver.log", required=True, ) parser.add_pattern("runtime", r"Singularity runtime: (.+?)s", type=float) parser.add_pattern("raw_memory", r"Peak memory: (\d+) KB", type=int) parser.add_pattern("cost", r"\nFinal value: (.+)\n", type=int) parser.add_function(coverage) parser.add_function(unsolvable) parser.add_function(error) parser.add_function(parse_g_value_over_time) parser.parse()
def main(): print("Running singularity parser") parser = Parser() parser.add_pattern( "planner_exit_code", r"run-planner exit code: (.+)\n", type=int, file="driver.log", required=True, ) parser.add_pattern( "node", r"node: (.+)\n", type=str, file="driver.log", required=True ) parser.add_pattern( "planner_wall_clock_time", r"run-planner wall-clock time: (.+)s", type=float, file="driver.log", required=True, ) parser.add_pattern("search_time", r"Search time: (.+)s", type=float) parser.add_pattern("total_time", r"Total time: (.+)s\n", type=float) # The Singularity runtime only has a granularity of seconds. parser.add_pattern("singularity_runtime", r"Singularity runtime: (.+)s", type=int) parser.add_pattern("raw_memory", r"Peak memory: (\d+) KB", type=int) parser.add_pattern("cost", r"\nFinal value: (.+)\n", type=int) parser.add_function(coverage) parser.add_function(unsolvable) parser.add_function(error) parser.parse()
def main(): print("Running singularity parser") parser = Parser() parser.add_pattern( "planner_exit_code", r"run-planner exit code: (.+)\n", type=int, file="driver.log", required=True, ) parser.add_pattern("node", r"node: (.+)\n", type=str, file="driver.log", required=True) parser.add_pattern( "planner_wall_clock_time", r"run-planner wall-clock time: (.+)s", type=float, file="driver.log", required=True, ) parser.add_pattern("runtime", r"Singularity runtime: (.+?)s", type=float) parser.add_pattern( "time_limit", r"Enforcing CPUTime limit \(soft limit, will send " r"SIGTERM then SIGKILL\): (\d+) seconds", type=int, file="watch.log", required=True, ) # Cumulative runtime and virtual memory of the solver and all child processes. parser.add_pattern("runtime", r"WCTIME=(.+)", type=float, file="values.log", required=True) parser.add_pattern("virtual_memory", r"MAXVM=(\d+)", type=int, file="values.log", required=True) parser.add_pattern("raw_memory", r"Peak memory: (\d+) KB", type=int) parser.add_pattern("cost", r"\nFinal value: (.+)\n", type=int) parser.add_function(coverage) parser.add_function(unsolvable) parser.add_function(parse_g_value_over_time) parser.add_function(set_outcome, file="values.log") parser.parse()
def main(): print "Running custom parser" parser = Parser() parser.add_pattern("pdb_collection_construction_time", "^PDB collection construction time: (.+)s$", type=float, flags="M", required=False) parser.add_pattern("dominance_pruning_time", "^Dominance pruning took (.+)s$", type=float, flags="M", required=False) parser.add_pattern("dominance_pruning_pruned_subsets", "Pruned (\d+) of \d+ maximal additive subsets", type=int, required=False) parser.add_pattern("dominance_pruning_pruned_pdbs", "Pruned (\d+) of \d+ PDBs", type=int, required=False) parser.add_function(add_dominance_pruning_failed) parser.parse()
def solved(content, props): props['solved'] = int('cover' in props) def error(content, props): if props['solved']: props['error'] = 'cover-found' else: props['error'] = 'unsolved' if __name__ == '__main__': parser = Parser() parser.add_pattern('node', r'node: (.+)\n', type=str, file='driver.log', required=True) parser.add_pattern('solver_exit_code', r'solve exit code: (.+)\n', type=int, file='driver.log') parser.add_pattern('cover', r'Cover: set\(\[(.*)\]\)', type=str) parser.add_pattern('cover_size', r'Cover size: (\d+)\n', type=int) parser.add_pattern('solve_time', r'Solve time: (.+)s', type=float) parser.add_function(solved) parser.add_function(error) parser.parse()
#!/usr/bin/env python import hashlib from lab.parser import Parser def add_hash_value(content, props): props['translator_output_sas_hash'] = hashlib.sha512(content).hexdigest() parser = Parser() parser.add_function(add_hash_value, file="output.sas") parser.parse()
type=int) parser.add_pattern('actual_search_time', 'Actual search time: (.+)s \[t=.+s\]', required=False, type=float) def check_ms_constructed(content, props): ms_construction_time = props.get('ms_construction_time') abstraction_constructed = False if ms_construction_time is not None: abstraction_constructed = True props['ms_abstraction_constructed'] = abstraction_constructed parser.add_function(check_ms_constructed) def check_planner_exit_reason(content, props): ms_abstraction_constructed = props.get('ms_abstraction_constructed') error = props.get('error') if error != 'none' and error != 'timeout' and error != 'out-of-memory': print 'error: %s' % error return # Check whether merge-and-shrink computation or search ran out of # time or memory. ms_out_of_time = False ms_out_of_memory = False search_out_of_time = False search_out_of_memory = False
parser = Parser() parser.add_pattern('ms_construction_time', 'Merge-and-shrink algorithm runtime: (.+)s', required=False, type=float) parser.add_pattern('ms_atomic_construction_time', 'M&S algorithm timer: (.+)s \(after computation of atomic factors\)', required=False, type=float) parser.add_pattern('ms_memory_delta', 'Final peak memory increase of merge-and-shrink algorithm: (\d+) KB', required=False, type=int) parser.add_pattern('ms_num_remaining_factors', 'Number of remaining factors: (\d+)', required=False, type=int) parser.add_pattern('ms_num_factors_kept', 'Number of factors kept: (\d+)', required=False, type=int) def check_ms_constructed(content, props): ms_construction_time = props.get('ms_construction_time') abstraction_constructed = False if ms_construction_time is not None: abstraction_constructed = True props['ms_abstraction_constructed'] = abstraction_constructed parser.add_function(check_ms_constructed) def check_atomic_fts_constructed(content, props): ms_atomic_construction_time = props.get('ms_atomic_construction_time') ms_atomic_fts_constructed = False if ms_atomic_construction_time is not None: ms_atomic_fts_constructed = True props['ms_atomic_fts_constructed'] = ms_atomic_fts_constructed parser.add_function(check_atomic_fts_constructed) def check_planner_exit_reason(content, props): ms_abstraction_constructed = props.get('ms_abstraction_constructed') error = props.get('error') if error != 'success' and error != 'timeout' and error != 'out-of-memory': print 'error: %s' % error
#print res if len(res) > 0 and props.get('plans') is None: props['unsolvable'] = 1 def total_time(content, props): p = re.findall(r'Total time: (.+)s$', content, re.M) props.pop('total_time', None) if len(p) > 0: props['total_time'] = max([float(i) for i in p]) parser = Parser() parser.add_pattern('evaluations', r'Evaluated (\d+) state\(s\).', required=False) parser.add_pattern('invertible', r'Number of invertible variables is (\d+)', required=False) parser.add_pattern('black', r'Total number of black variables is (\d+)', required=False) parser.add_function(get_plans) parser.add_function(get_best_plan) parser.add_function(unsolvable) parser.add_function(coverage) parser.add_function(total_time) parser.parse()
props['false_pos_novelty'] = false_pos_novelty props['tuple_false_pos'] = tuple_false_pos props['h1'] = h1 props['h2'] = h2 flag_new_value = False cost = None generated = None expanded = None runtime = None memory = None plan = None goal = None width = None false_pos_novelty = None tuple_false_pos = None h1 = None h2 = None # parser = Parser() parser.add_pattern('planner_exit_code', r'solve exit code: (.+)\n', type=int, file='driver.log') parser.add_function(fetch_props, 'run.log') parser.add_function(modify_props_for_atomic_goals, 'run.log') parser.parse()
#! /usr/bin/env python import math from lab.parser import Parser def diff(content, props): pi = props.get('pi') props['diff'] = abs(math.pi - pi) parser = Parser() parser.add_pattern('pi', 'Pi: (.+)', type=float) parser.add_pattern('time', 'Time: (.+)', type=float) parser.add_function(diff) parser.parse()
def get_num_novel_features_complexity_5(content, props): # All patterns are parsed before functions are called. if props.get("generate_time_complexity_5") is not None: props["num_novel_features_complexity_5"] = int(re.findall(r"Complexity 5:\nTotal generated features: .+\nTotal novel features: (.+)", content, re.M)[0]) def get_num_generated_features_complexity_10(content, props): # All patterns are parsed before functions are called. if props.get("generate_time_complexity_10") is not None: props["num_generated_features_complexity_10"] = int(re.findall(r"Complexity 10:\nTotal generated features: (.+)", content, re.M)[0]) def get_num_novel_features_complexity_10(content, props): # All patterns are parsed before functions are called. if props.get("generate_time_complexity_10") is not None: props["num_novel_features_complexity_10"] = int(re.findall(r"Complexity 10:\nTotal generated features: .+\nTotal novel features: (.+)", content, re.M)[0]) parser = Parser() parser.add_pattern("generate_time_complexity_5", r"\[t=(.+)s, .+ KB\] Complexity 5:\n", type=float) parser.add_pattern("generate_memory_complexity_5", r"\[t=.+s, (.+) KB\] Complexity 5:\n", type=int) parser.add_pattern("generate_time_complexity_10", r"\[t=(.+)s, .+ KB\] Complexity 10:\n", type=float) parser.add_pattern("generate_memory_complexity_10", r"\[t=.+s, (.+) KB\] Complexity 10:\n", type=int) parser.add_pattern("num_states", r"Number of states: (.+)", type=int) parser.add_pattern("num_dynamic_atoms", r"Number of dynamic atoms: (.+)", type=int) parser.add_pattern("num_static_atoms", r"Number of static atoms: (.+)", type=int) parser.add_pattern("evaluate_time", r"Time to construct and evaluate features: (.+)s", type=float) parser.add_function(get_num_generated_features_complexity_5) parser.add_function(get_num_novel_features_complexity_5) parser.add_function(get_num_generated_features_complexity_10) parser.add_function(get_num_novel_features_complexity_10) parser.parse()
#! /usr/bin/env python import math from lab.parser import Parser def diff(content, props): pi = props.get("pi") props["diff"] = abs(math.pi - pi) parser = Parser() parser.add_pattern("pi", "Pi: (.+)", type=float) parser.add_pattern("time", "Time: (.+)", type=float) parser.add_function(diff) parser.parse()
#---- Bloom-sampling algorithm specific x = re.search(r'Bloom Fillratio: ([\d.e-]+)', content) props['bloom_fillratio'] = float(x.group(1)) if x else None x = re.search(r'Novelty-Num_nodes Map: ([^\n]+)', content) props['novelty:num_nodes'] = x.group(1) if x else None #xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx# #-----------------------------------------------------------------------------# def coverage(content, props) : props['coverage'] = int(props['is_plan_valid']==True) #xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx# #-----------------------------------------------------------------------------# def parse_plan(content, props) : x = re.match(r'.*', content, re.DOTALL) props['plan'] = x.group(0) if x else none props['plan_file'] = abspath('plan.ipc') #xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx# parser = Parser() parser.add_pattern('solver_exit_code', r'solve exit code: (.+)\n', type=int, file='driver.log') parser.add_pattern('validate_exit_code', r'validate exit code: (.+)\n', type=int, file='driver.log') parser.add_pattern('lab_exit_code', r'lab_parser exit code: (.+)\n', type=int, file='driver.log') parser.add_function(fetch_props,'run.log') parser.add_function(parse_plan,'plan.ipc') parser.add_function(coverage) parser.parse()
props['coverage'] = int(props['planner_exit_code'] == 0) def get_plan(content, props): # All patterns are parsed before functions are called. if props.get('evaluations') is not None: props['plan'] = re.findall(r'^(?:step)?\s*\d+: (.+)$', content, re.M) def get_times(content, props): props['times'] = re.findall(r'(\d+\.\d+) seconds', content) def trivially_unsolvable(content, props): props['trivially_unsolvable'] = int( 'ff: goal can be simplified to FALSE. No plan will solve it' in content) parser = Parser() parser.add_pattern( 'node', r'node: (.+)\n', type=str, file='driver.log', required=True) parser.add_pattern( 'planner_exit_code', r'run-planner exit code: (.+)\n', type=int, file='driver.log') parser.add_pattern('evaluations', r'evaluating (\d+) states') parser.add_function(error) parser.add_function(coverage) parser.add_function(get_plan) parser.add_function(get_times) parser.add_function(trivially_unsolvable) parser.parse()
#! /usr/bin/env python from lab.parser import Parser def wordcount(content, props): props['number_of_files'] = len(content.splitlines()) parser = Parser() parser.add_function(wordcount) parser.add_pattern('first_number', '(\d+)') parser.parse()
from lab.parser import Parser parser = Parser() parser.add_pattern('initial_h_value', 'initial h value: (\d+)', required=False, type=int) parser.add_pattern('ms_final_size', 'Final transition system size: (\d+)', required=False, type=int) parser.add_pattern('ms_construction_time', 'Done initializing merge-and-shrink heuristic \[(.+)s\]', required=False, type=float) def check_ms_constructed(content, props): ms_construction_time = props.get('ms_construction_time') abstraction_constructed = False if ms_construction_time is not None: abstraction_constructed = True props['ms_abstraction_constructed'] = abstraction_constructed parser.add_function(check_ms_constructed) def check_proved_unsolvability(content, props): proved_unsolvability = False if props['coverage'] == 0: for line in content.splitlines(): if line == 'Completely explored state space -- no solution!': proved_unsolvability = True break props['proved_unsolvability'] = proved_unsolvability parser.add_function(check_proved_unsolvability) def check_planner_exit_reason(content, props): ms_abstraction_constructed = props.get('ms_abstraction_constructed') error = props.get('error')
from lab.parser import Parser parser = Parser() parser.add_pattern('ms_final_size', 'Final transition system size: (\d+)', required=False, type=int) parser.add_pattern('ms_construction_time', 'Done initializing merge-and-shrink heuristic \[(.+)s\]', required=False, type=float) parser.add_pattern('ms_memory_delta', 'Final peak memory increase of merge-and-shrink computation: (\d+) KB', required=False, type=int) parser.add_pattern('actual_search_time', 'Actual search time: (.+)s \[t=.+s\]', required=False, type=float) def check_ms_constructed(content, props): ms_construction_time = props.get('ms_construction_time') abstraction_constructed = False if ms_construction_time is not None: abstraction_constructed = True props['ms_abstraction_constructed'] = abstraction_constructed parser.add_function(check_ms_constructed) def check_planner_exit_reason(content, props): ms_abstraction_constructed = props.get('ms_abstraction_constructed') error = props.get('error') if error != 'none' and error != 'timeout' and error != 'out-of-memory': print 'error: %s' % error return # Check whether merge-and-shrink computation or search ran out of # time or memory. ms_out_of_time = False ms_out_of_memory = False search_out_of_time = False search_out_of_memory = False if ms_abstraction_constructed == False:
#! /usr/bin/env python from lab.parser import Parser def wordcount(content, props): props["number_of_files"] = len(content.splitlines()) parser = Parser() parser.add_function(wordcount) parser.add_pattern("first_number", "(\d+)") parser.parse()
type=float) parser.add_pattern('cpdbs_time', 'PDB collection construction time: (.+)s', required=False, type=float) def check_hc_constructed(content, props): hc_time = props.get('hc_total_time') abstraction_constructed = False if hc_time is not None: abstraction_constructed = True props['hc_abstraction_constructed'] = abstraction_constructed parser.add_function(check_hc_constructed) def check_planner_exit_reason(content, props): hc_abstraction_constructed = props.get('hc_abstraction_constructed') error = props.get('error') if error != 'none' and error != 'timeout' and error != 'out-of-memory': print 'error: %s' % error return # Check whether hill climbing computation or search ran out of # time or memory. hc_out_of_time = False hc_out_of_memory = False search_out_of_time = False search_out_of_memory = False
def main(): parser = Parser() parser.add_pattern("raw_memory", r"Peak memory: (.+) KB", type=int), parser.add_function(find_all_matches("cost:all", r"Plan cost: (.+)\n", type=float)) parser.add_function( find_all_matches("steps:all", r"Plan length: (.+) step\(s\).\n", type=float) ) parser.add_function(reduce_to_min("cost:all", "cost")) parser.add_function(reduce_to_min("steps:all", "steps")) parser.add_function(coverage) parser.add_function(add_memory) parser.parse()
#! /usr/bin/env python from lab.parser import Parser def no_search(content, props): # Patterns for time and memory have the same structure, # they match the same locations but extract different data if "search_start_time" not in props: error = props.get("error") if error is not None and error != "incomplete-search-found-no-plan": props["error"] = "no-search-due-to-" + error print("Running start time/memory parser") parser = Parser() parser.add_pattern("search_start_time", r"\[g=0, 1 evaluated, 0 expanded, t=(.+)s, \d+ KB\]", type=float) parser.add_pattern("search_start_memory", r"\[g=0, 1 evaluated, 0 expanded, t=.+s, (\d+) KB\]", type=int) parser.add_function(no_search) parser.parse()