def parse(context, filename): """Parse a .test file as used in the llvm test-suite. The file comporises of a number of lines starting with RUN: and VERIFY: specifying shell commands to run the benchmark and verifying the result. Returns a tuple with two arrays for the run and verify commands.""" # Collect the test lines from the script. preparescript = [] runscript = [] verifyscript = [] metricscripts = {} # Note that we keep both "RUN" and "RUN:" in the list to stay compatible # with older lit versions. keywords = ['PREPARE:', 'PREPARE', 'RUN:', 'RUN', 'VERIFY:', 'VERIFY', 'METRIC:', 'METRIC'] for line_number, command_type, ln in \ parseIntegratedTestScriptCommands(filename, keywords): if command_type.startswith('PREPARE'): _parseShellCommand(preparescript, ln) elif command_type.startswith('RUN'): _parseShellCommand(runscript, ln) elif command_type.startswith('VERIFY'): _parseShellCommand(verifyscript, ln) elif command_type.startswith('METRIC'): metric, ln = ln.split(':', 1) metricscript = metricscripts.setdefault(metric.strip(), list()) _parseShellCommand(metricscript, ln) else: raise ValueError("unknown script command type: %r" % ( command_type,)) # Verify the script contains a run line. if runscript == []: raise ValueError("Test has no RUN: line!") # Check for unterminated run lines. for script in preparescript, runscript, verifyscript: if script and script[-1][-1] == '\\': raise ValueError("Test has unterminated RUN/VERIFY lines " + "(ending with '\\')") # Apply the usual lit substitutions (%s, %S, %p, %T, ...) outfile = context.tmpBase + ".out" substitutions = getDefaultSubstitutions(context.test, context.tmpDir, context.tmpBase) substitutions += [('%o', outfile)] preparescript = applySubstitutions(preparescript, substitutions) runscript = applySubstitutions(runscript, substitutions) verifyscript = applySubstitutions(verifyscript, substitutions) metricscripts = {k: applySubstitutions(v, substitutions) for k, v in metricscripts.items()} # Put things into the context context.parsed_preparescript = preparescript context.parsed_runscript = runscript context.parsed_verifyscript = verifyscript context.parsed_metricscripts = metricscripts context.executable = shellcommand.getMainExecutable(context) if not context.executable: logging.error("Could not determine executable name in %s" % filename)
def parse(context, filename): """Parse a .test file as used in the llvm test-suite. The file comporises of a number of lines starting with RUN: and VERIFY: specifying shell commands to run the benchmark and verifying the result. Returns a tuple with two arrays for the run and verify commands.""" # Collect the test lines from the script. preparescript = [] runscript = [] verifyscript = [] metricscripts = {} keywords = ['PREPARE:', 'RUN:', 'VERIFY:', 'METRIC:'] for line_number, command_type, ln in \ parseIntegratedTestScriptCommands(filename, keywords): if command_type == 'PREPARE': _parseShellCommand(preparescript, ln) elif command_type == 'RUN': _parseShellCommand(runscript, ln) elif command_type == 'VERIFY': _parseShellCommand(verifyscript, ln) elif command_type == 'METRIC': metric, ln = ln.split(':', 1) metricscript = metricscripts.setdefault(metric.strip(), list()) _parseShellCommand(metricscript, ln) else: raise ValueError("unknown script command type: %r" % ( command_type,)) # Verify the script contains a run line. if runscript == []: raise ValueError("Test has no RUN: line!") # Check for unterminated run lines. for script in preparescript, runscript, verifyscript: if script and script[-1][-1] == '\\': raise ValueError("Test has unterminated RUN/VERIFY lines " + "(ending with '\\')") # Apply the usual lit substitutions (%s, %S, %p, %T, ...) outfile = context.tmpBase + ".out" substitutions = getDefaultSubstitutions(context.test, context.tmpDir, context.tmpBase) substitutions += [('%o', outfile)] preparescript = applySubstitutions(preparescript, substitutions) runscript = applySubstitutions(runscript, substitutions) verifyscript = applySubstitutions(verifyscript, substitutions) metricscripts = {k: applySubstitutions(v, substitutions) for k, v in metricscripts.items()} # Put things into the context context.parsed_preparescript = preparescript context.parsed_runscript = runscript context.parsed_verifyscript = verifyscript context.parsed_metricscripts = metricscripts context.executable = shellcommand.getMainExecutable(context) if not context.executable: logging.error("Could not determine executable name in %s" % filename)
def execute(self, test, litConfig): config = test.config if config.unsupported: return lit.Test.Result(Test.UNSUPPORTED, 'Test is unsupported') # Parse benchmark script plan = testplan.parse(test.getSourcePath()) if litConfig.noExecute: return lit.Test.Result(Test.PASS) # Apply the usual lit substitutions (%s, %S, %p, %T, ...) tmpDir, tmpBase = getTempPaths(test) outfile = tmpBase + ".out" substitutions = getDefaultSubstitutions(test, tmpDir, tmpBase) substitutions += [('%o', outfile)] plan.runscript = applySubstitutions(plan.runscript, substitutions) plan.verifyscript = applySubstitutions(plan.verifyscript, substitutions) plan.metricscripts = {k: applySubstitutions(v, substitutions) for k, v in plan.metricscripts.items()} context = TestContext(test, litConfig, plan.runscript, plan.verifyscript, tmpDir, tmpBase) context.executable = shellcommand.getMainExecutable(context) if context.executable is None: return lit.Test.Result(Test.UNSUPPORTED, 'Could not determine executable name') hash.compute(context) if hash.same_as_previous(context): return lit.Test.Result(SKIPPED, 'Executable identical to previous run') # Create the output directory if it does not already exist. lit.util.mkdir_p(os.path.dirname(tmpBase)) # Prepare test plan run_under.mutatePlan(context, plan) timeit.mutatePlan(context, plan) compiletime.mutatePlan(context, plan) codesize.mutatePlan(context, plan) hash.mutatePlan(context, plan) if config.profile_generate: profilegen.mutatePlan(context, plan) if config.remote_host: remote.mutatePlan(context, plan) if litConfig.params.get('profile') == 'perf': perf.mutatePlan(context, plan) # Execute Test plan result = testplan.executePlanTestResult(context, plan) return result