def main(): parser = argparse.ArgumentParser( description='Build a project using CMake and Makefiles') # required arguments parser.add_argument('--src-dir', nargs=1, required=True, help='Directory containing the source to build') parser.add_argument('--build-dir', nargs=1, required=True, help='Directory to hold the build files') parser.add_argument('--install-dir', nargs=1, required=True, help='Directory to hold the installed files') parser.add_argument('--cc', nargs=1, required=True, help='Command to use to compile C source') parser.add_argument('--cxx', nargs=1, required=True, help='Command to use to compile C++ source') parser.add_argument('--fort', nargs=1, required=True, help='Command to use to compile Fortran source') # optional arguments parser.add_argument('--build-flags', nargs=1, required=True, help='Argument to be used in all compilations') parser.set_defaults(build_flags=['']) args = parser.parse_args(sys.argv[1:]) src_dir = os.path.abspath(args.src_dir[0]) build_dir = os.path.abspath(args.build_dir[0]) install_dir = os.path.abspath(args.install_dir[0]) cc = args.cc[0] cxx = args.cxx[0] fort = args.fort[0] build_flags = args.build_flags[0] if not os.path.exists(src_dir): eprint ('-- Source directory \'' + src_dir + '\' does not exist') return -1 if not os.path.exists(build_dir): eprint ('-- Build directory \'' + build_dir + '\' does not exist') return -1 if not os.path.exists(install_dir): eprint ('-- Install directory \'' + install_dir + '\' does not exist') return -1 if not mageec.is_command_on_path(cc): eprint ('-- Compiler \'' + cc + '\' is not on the path') return -1 if not mageec.is_command_on_path(cxx): eprint ('-- Compiler \'' + cxx + '\' is not on the path') return -1 if not mageec.is_command_on_path(fort): eprint ('-- Compiler \'' + fort + '\' is not on the path') return -1 res = build(src_dir, build_dir, install_dir, cc, cxx, fort, build_flags) if not res: return -1 return 0
def generate_configurations(src_dir, build_dir, install_dir, build_system, cc, cxx, fort, flags, jobs, database_path, features_path, num_configs, generator, debug): assert (os.path.exists(src_dir) and os.path.isabs(src_dir)) assert (os.path.exists(build_dir) and os.path.isabs(build_dir)) assert (os.path.exists(install_dir) and os.path.isabs(install_dir)) assert (os.path.exists(database_path)) assert (os.path.exists(features_path)) assert (mageec.is_command_on_path(cc)) assert (mageec.is_command_on_path(cxx)) assert (mageec.is_command_on_path(fort)) assert (num_configs > 0) assert (jobs > 0) configs = generate_configs(gcc_flags, num_configs, generator) run_id = 0 for config in configs: run_build_dir = os.path.join(build_dir, 'run-' + str(run_id)) run_install_dir = os.path.join(install_dir, 'run-' + str(run_id)) if not os.path.exists(run_build_dir): os.makedirs(run_build_dir) if not os.path.exists(run_install_dir): os.makedirs(run_install_dir) run_id += 1 print('-- Building configuration:\n' ' Configuration: \'' + config + '\'') compilations_path = os.path.join(run_install_dir, 'compilations.csv') cc_wrapper = 'mageec-' + cc cxx_wrapper = 'mageec-' + cxx fort_wrapper = 'mageec-' + fort assert (mageec.is_command_on_path(cc_wrapper)) assert (mageec.is_command_on_path(cxx_wrapper)) assert (mageec.is_command_on_path(fort_wrapper)) wrapper_flags = "" if debug: wrapper_flags += ' -fmageec-debug' wrapper_flags += ' -fmageec-mode=gather' wrapper_flags += ' -fmageec-database=' + database_path wrapper_flags += ' -fmageec-features=' + features_path wrapper_flags += ' -fmageec-out=' + compilations_path new_flags = wrapper_flags + ' ' + flags + ' ' + config res = mageec.build(src_dir=src_dir, build_dir=run_build_dir, install_dir=run_install_dir, build_system=build_system, cc=cc_wrapper, cxx=cxx_wrapper, fort=fort_wrapper, flags=new_flags) # just ignore failed builds if not res: print('-- Build failed. Continuing regardless') return True
def generate_configurations(src_dir, build_dir, install_dir, build_system, cc, cxx, fort, flags, jobs, database_path, features_path, num_configs, generator, debug): assert(os.path.exists(src_dir) and os.path.isabs(src_dir)) assert(os.path.exists(build_dir) and os.path.isabs(build_dir)) assert(os.path.exists(install_dir) and os.path.isabs(install_dir)) assert(os.path.exists(database_path)) assert(os.path.exists(features_path)) assert(mageec.is_command_on_path(cc)) assert(mageec.is_command_on_path(cxx)) assert(mageec.is_command_on_path(fort)) assert(num_configs > 0) assert(jobs > 0) configs = generate_configs(gcc_flags, num_configs, generator) run_id = 0 for config in configs: run_build_dir = os.path.join(build_dir, 'run-' + str(run_id)) run_install_dir = os.path.join(install_dir, 'run-' + str(run_id)) if not os.path.exists(run_build_dir): os.makedirs(run_build_dir) if not os.path.exists(run_install_dir): os.makedirs(run_install_dir) run_id += 1 print ('-- Building configuration:\n' ' Configuration: \'' + config + '\'') compilations_path = os.path.join(run_install_dir, 'compilations.csv') cc_wrapper = 'mageec-' + cc cxx_wrapper = 'mageec-' + cxx fort_wrapper = 'mageec-' + fort assert(mageec.is_command_on_path(cc_wrapper)) assert(mageec.is_command_on_path(cxx_wrapper)) assert(mageec.is_command_on_path(fort_wrapper)) wrapper_flags = "" if debug: wrapper_flags += ' -fmageec-debug' wrapper_flags += ' -fmageec-mode=gather' wrapper_flags += ' -fmageec-database=' + database_path wrapper_flags += ' -fmageec-features=' + features_path wrapper_flags += ' -fmageec-out=' + compilations_path new_flags = wrapper_flags + ' ' + flags + ' ' + config res = mageec.build(src_dir=src_dir, build_dir=run_build_dir, install_dir=run_install_dir, build_system=build_system, cc=cc_wrapper, cxx=cxx_wrapper, fort=fort_wrapper, flags=new_flags) # just ignore failed builds if not res: print ('-- Build failed. Continuing regardless') return True
def build(src_dir, build_dir, install_dir, cc, cxx, fort, build_flags): assert (mageec.is_command_on_path('cmake')) assert (mageec.is_command_on_path('make')) base_dir = os.getcwd() with mageec.preserve_cwd(): os.chdir(build_dir) cmd = ['cmake', src_dir, '-G', 'Unix Makefiles'] cmd.append('-DCMAKE_C_COMPILER=' + cc) cmd.append('-DCMAKE_CXX_COMPILER=' + cxx) if build_flags != '': cmd.append('-DCMAKE_C_FLAGS=' + build_flags) cmd.append('-DCMAKE_CXX_FLAGS=' + build_flags) cmd.append('-DCMAKE_INSTALL_PREFIX=' + install_dir) # configure the build print (' '.join(cmd)) ret = subprocess.call(cmd) if ret != 0: eprint ('-- Failed to configure \'' + src_dir + '\' using CMake') return False # Call make to do the build cmd = ['make'] print (' '.join(cmd)) ret = subprocess.call(cmd) if ret != 0: eprint ('-- Failed to build source \'' + src_dir + '\' using CMake') return False cmd = ['make', 'install'] print (' '.join(cmd)) ret = subprocess.call(cmd) if ret != 0: eprint ('-- Failed to install build \'' + build_dir + '\' to \'' + install_dir) return False return True
def build(src_dir, build_dir, install_dir, cc, cxx, fort, build_flags): assert (mageec.is_command_on_path('cmake')) assert (mageec.is_command_on_path('make')) base_dir = os.getcwd() with mageec.preserve_cwd(): os.chdir(build_dir) cmd = [os.path.join(src_dir, 'configure')] cmd.append('CC=' + cc) cmd.append('CXX=' + cxx) if build_flags != '': cmd.append('CFLAGS=' + build_flags) cmd.append('CXXFLAGS=' + build_flags) cmd.append('--prefix=' + install_dir) # configure the build print (' '.join(cmd)) ret = subprocess.call(cmd) if ret != 0: eprint ('-- Failed to configure \'' + src_dir + '\' using CMake') return False # Call make to do the build and install cmd = ['make'] print (' '.join(cmd)) ret = subprocess.call(cmd) if ret != 0: eprint ('-- Failed to build source \'' + src_dir + '\' using CMake') return False cmd = ['make', 'install'] print (' '.join(cmd)) ret = subprocess.call(cmd) if ret != 0: eprint ('-- Failed to install build \'' + build_dir + '\' to \'' + install_dir) return False return True
def main(): parser = argparse.ArgumentParser( description='Perform feature extraction') # required arguments parser.add_argument('--src-dir', nargs=1, required=True, help='Directory containing the source to build') parser.add_argument('--build-dir', nargs=1, required=True, help='Build directory') parser.add_argument('--install-dir', nargs=1, required=True, help='Install directory') parser.add_argument('--cc', nargs=1, required=True, help='Command to use to compile C source') parser.add_argument('--cxx', nargs=1, required=True, help='Command to use to compile C++ source') parser.add_argument('--fort', nargs=1, required=True, help='Command to use to compile Fortan source') parser.add_argument('--database', nargs=1, required=True, help='mageec database to store extracted features into') parser.add_argument('--mageec-lib-path', nargs=1, required=True, help='Path to the directory holding the mageec libraries') parser.add_argument('--build-script', nargs=1, required=True, help='Script to build the benchmarks') parser.add_argument('--out', nargs=1, required=True, help='File to output the extracted features to') # optional arguments parser.add_argument('--debug', action='store_true', required=False, help='Enable debug when doing feature extraction') parser.add_argument('--build-flags', nargs=1, required=False, help='Common arguments to be used when building') parser.set_defaults(debug=False, build_flags=['']) args = parser.parse_args(sys.argv[1:]) src_dir = os.path.abspath(args.src_dir[0]) build_dir = os.path.abspath(args.build_dir[0]) install_dir = os.path.abspath(args.install_dir[0]) cc = args.cc[0] cxx = args.cxx[0] fort = args.fort[0] database = os.path.abspath(args.database[0]) mageec_lib_path = os.path.abspath(args.mageec_lib_path[0]) build_script = args.build_script[0] out = os.path.abspath(args.out[0]) debug = args.debug build_flags = args.build_flags[0] # TODO: Extension should depend on platform gcc_plugin_name = mageec.get_gcc_plugin_name() + '.so' if not os.path.exists(src_dir): eprint ('-- Source directory \'' + src_dir + '\' does not exist') return -1 if not os.path.exists(database): eprint ('-- Database \'' + database + '\' does not exist') return -1 if os.path.exists(build_dir): eprint ('-- Build directory \'' + build_dir + '\' already exists') return -1 if os.path.exists(install_dir): eprint ('-- Install directory \'' + install_dir + '\' already exists') return -1 if not mageec.is_command_on_path(cc): eprint ('-- Compiler \'' + cc + '\' is not on the path') return -1 if not mageec.is_command_on_path(cxx): eprint ('-- Compiler \'' + cxx + '\' is not on the path') return -1 if not mageec.is_command_on_path(fort): eprint ('-- Compiler \'' + fort + '\' is not on the path') return -1 print ('-- Checking for mageec plugin \'' + gcc_plugin_name + '\'') gcc_plugin_path = os.path.join(mageec_lib_path, gcc_plugin_name) if not os.path.exists(gcc_plugin_path): eprint ('-- Could not find gcc plugin') return -1 os.makedirs(build_dir) os.makedirs(install_dir) res = mageec.feature_extract(src_dir=src_dir, build_dir=build_dir, install_dir=install_dir, cc=cc, cxx=cxx, fort=fort, database=database, build_script=build_script, build_flags=build_flags, gcc_plugin_path=gcc_plugin_path, debug=debug, out=out) if not res: return -1 return 0
def main(): parser = argparse.ArgumentParser( description='Generate and build multiple versions of a source project') # required arguments parser.add_argument('--src-dir', nargs=1, required=True, help='Directory containing the source to build') parser.add_argument('--run-dir', nargs=1, required=True, help='Directory to hold each combined elimination run') parser.add_argument('--cc', nargs=1, required=True, help='Command to use to compile C source') parser.add_argument('--cxx', nargs=1, required=True, help='Command to use to compile C++ source') parser.add_argument('--fort', nargs=1, required=True, help='Command to use to compile Fortran source') parser.add_argument('--database', nargs=1, required=True, help='mageec database to store generated compilations in') parser.add_argument('--features', nargs=1, required=True, help='File containing extracted features for the source being built') parser.add_argument('--base-opt', nargs=1, required=True, help='Base optimization level to use as a starting point. ' 'Valid values are \'-O3\' and \'-Os\'') parser.add_argument('--build-script', nargs=1, required=True, help='Script to build the benchmarks') parser.add_argument('--measure-script', nargs=1, required=True, help='Script to measure the resultant executables') # optional arguments parser.add_argument('--build-flags', nargs=1, required=False, help='Argument to be used in all compilations') parser.add_argument('--exec-flags', nargs=1, required=False, help='Flags to use when executing generated programs') parser.add_argument('--jobs', nargs=1, required=False, help='Number of jobs to run when building') parser.add_argument('--debug', action='store_true', required=False, help='Enable debug when doing feature extraction') parser.set_defaults(build_flags=[''], exec_flags=[''], jobs=[1], debug=False) args = parser.parse_args(sys.argv[1:]) src_dir = os.path.abspath(args.src_dir[0]) run_dir = os.path.abspath(args.run_dir[0]) cc = args.cc[0] cxx = args.cxx[0] fort = args.fort[0] database = os.path.abspath(args.database[0]) features = os.path.abspath(args.features[0]) base_opt = args.base_opt[0] build_script = args.build_script[0] measure_script = args.measure_script[0] build_flags = args.build_flags[0] exec_flags = args.exec_flags[0] jobs = int(args.jobs[0]) debug = args.debug if not os.path.exists(src_dir): eprint ('-- Source directory \'' + src_dir + '\' does not exist') return -1 if not os.path.exists(database): eprint ('-- Database \'' + database + '\' does not exist') return -1 if not os.path.exists(features): eprint ('-- Features file \'' + features + '\' does not exist') return -1 if not mageec.is_command_on_path(cc): eprint ('-- Compiler \'' + cc + '\' is not on the path') return -1 if not mageec.is_command_on_path(cxx): eprint ('-- Compiler \'' + cxx + '\' is not on the path') return -1 if not mageec.is_command_on_path(fort): eprint ('-- Compiler \'' + fort + '\' is not on the path') return -1 res = combined_elimination(src_dir=src_dir, run_dir=run_dir, cc=cc, cxx=cxx, fort=fort, database=database, features=features, build_script=build_script, measure_script=measure_script, base_opt=base_opt, build_flags=build_flags, exec_flags=exec_flags, jobs=jobs, debug=debug) if not res: return -1 return 0
def combined_elimination(src_dir, run_dir, cc, cxx, fort, database, features, build_script, measure_script, base_opt, build_flags, exec_flags, jobs, debug): assert(os.path.exists(src_dir) and os.path.isabs(src_dir)) assert(os.path.exists(run_dir) and os.path.isabs(run_dir)) assert(mageec.is_command_on_path(cc)) assert(mageec.is_command_on_path(cxx)) assert(mageec.is_command_on_path(fort)) assert(os.path.exists(database)) assert(os.path.exists(features)) assert(jobs > 0) # Get the version of the compiler and determine which flags are available # for tuning cmd = [cc, '-dumpversion'] print (' '.join(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE) gcc_major, gcc_minor, gcc_patch = p.communicate()[0].decode().strip().split('.') gcc_version = (int(gcc_major) * 10000) + (int(gcc_minor) * 100) + int(gcc_patch) all_flags = [] for flag in flag_version: version = flag_version[flag] if version <= gcc_version: all_flags.append(flag) # Flags to consider as candidates to be disabled. This list will shrink # when flags are disabled permanently flags_to_consider = list(all_flags) # Build at the base optimization level. This baseline build isn't actually # run, but can be used as a point of comparison later. assert(base_opt.strip() in ['-O3', '-Os']) base_build_dir = os.path.join(run_dir, 'base', 'build') base_install_dir = os.path.join(run_dir, 'base', 'install') assert(not os.path.exists(base_build_dir)) assert(not os.path.exists(base_install_dir)) os.makedirs(base_build_dir) os.makedirs(base_install_dir) res = mageec.build(src_dir=src_dir, build_dir=base_build_dir, install_dir=base_install_dir, build_script=build_script, cc=cc, cxx=cxx, fort=fort, build_flags=build_flags + ' ' + base_opt) if not res: eprint ('-- Failed baseline build') return False # Identifier for the current run run_id = 0 # Begin with all flags enabled curr_flags = list(all_flags) curr_result = build_and_measure(src_dir, os.path.join(run_dir, 'test.' + str(run_id)), cc, cxx, fort, database, features, build_script, measure_script, [build_flags, base_opt] + curr_flags, [exec_flags], debug) if curr_result <= 0: eprint ('-- Failed initial test run') return False print ('CE: (best) id: ' + str(run_id) + ' result: ' + str(curr_result) + ' flags: ' + ' '.join(curr_flags)) run_id += 1 finished = False while not finished: finished = True # identify flags that bring improvement when they are disabled # individually. Do all of these builds in parallel flags_with_improvement = [] test_pool = Pool(jobs) test_runs = [] for test_flag in flags_to_consider: run_flags = list(curr_flags) # Flip the flag to have a -fno- prefix assert(test_flag in run_flags) flag_index = run_flags.index(test_flag) run_flags[flag_index] = '-fno-' + test_flag[2:] # Build and measure with this flag disabled run_proc = test_pool.apply_async(build_and_measure, (src_dir, os.path.join(run_dir, 'test.' + str(run_id)), cc, cxx, fort, database, features, build_script, measure_script, [build_flags, base_opt] + run_flags, [exec_flags], debug)) test_runs.append((run_proc, run_id, test_flag)) run_id += 1 # Wait for all of the test runs to complete, and then get their results for run_proc, _, _ in test_runs: run_proc.wait() for run_proc, test_run_id, test_flag in test_runs: run_result = run_proc.get() if run_result <= 0: eprint ('-- Ignoring failed test run ' + str(test_run_id)) continue print ('CE: (test) id: ' + str(test_run_id) + ' result: ' + str(run_result) + ' flag: ' + test_flag) # Consider a flag as being an improvement even if it's up to 1% # worse than the current base result. This avoids small # measurement errors from disrupting the process. if run_result < (curr_result * 1.01): flags_with_improvement.append((test_flag, test_run_id, run_result)) # Starting from the flag which had the biggest improvement, check # if disabling the flag still gives a better result. If it does, then # permanently disable the flag and update the base flag # configuration. # # The first flag is guaranteed to give an improvement, so we don't # need to re-run the test flags_with_improvement = sorted(flags_with_improvement, key=lambda x: x[2]) if len(flags_with_improvement) == 0: break # If the first flag really is better than the current best, then # immediately invert it and remove it from further consideration test_flag, test_run_id, run_result = flags_with_improvement[0] if run_result < curr_result: assert test_flag in curr_flags flag_index = curr_flags.index(test_flag) curr_flags[flag_index] = '-fno-' + test_flag[2:] curr_result = run_result print ('CE: (best) id: ' + str(test_run_id) + ' result: ' + str(curr_result) + ' flags: ' + ' '.join(curr_flags)) flags_to_consider.remove(test_flag) flags_with_improvement = flags_with_improvement[1:] # Keep disabling flags which previously gave an improvement # in turn until we stop seeing an improvement for flag, _, _ in flags_with_improvement: run_flags = list(curr_flags) # Flip the flag to have a -fno- prefix assert flag in run_flags flag_index = run_flags.index(flag) run_flags[flag_index] = '-fno-' + flag[2:] # Run with this flag disabled as well test_run_id = run_id run_result = build_and_measure(src_dir, os.path.join(run_dir, 'test.' + str(run_id)), cc, cxx, fort, database, features, build_script, measure_script, [build_flags, base_opt] + run_flags, [exec_flags], debug) print ('CE: (test) id: ' + str(test_run_id) + ' result: ' + str(run_result) + ' flag: ' + flag) run_id += 1 if run_result <= 0: eprint ('-- Ignoring failed test run ' + str(test_run_id)) # TODO: If this flag is within 1% of the current result, then # rerun a couple more times in case it's just a measurement # anomaly if run_result < curr_result: # Disabling this flag was an improvement, so permanently # disable it and remove it from further consideration curr_flags = run_flags curr_result = run_result print ('CE: (best) id: ' + str(test_run_id) + ' result: ' + str(curr_result) + ' flags: ' + ' '.join(curr_flags)) flags_to_consider.remove(flag) # Another iteration of combined elimination is necessary finished = False
def build_and_measure(src_dir, run_dir, cc, cxx, fort, database, features, build_script, measure_script, build_flags, exec_flags, debug): assert(os.path.exists(src_dir)) assert(os.path.exists(database)) assert(os.path.exists(features)) build_dir = os.path.join(run_dir, 'build') install_dir = os.path.join(run_dir, 'install') assert(not os.path.exists(build_dir)) assert(not os.path.exists(install_dir)) os.makedirs(build_dir) os.makedirs(install_dir) # build the benchmark using the wrapper driver # This will record the flags which the benchmark is being compiled with, # and associate these flags with previously extracted features through # an integer identifier of the compilation in the database compilations = os.path.join(install_dir, 'compilations.csv') results_path = os.path.join(install_dir, 'results.csv') cc_wrapper = 'mageec-' + cc cxx_wrapper = 'mageec-' + cxx fort_wrapper = 'mageec-' + fort assert(mageec.is_command_on_path(cc_wrapper)) assert(mageec.is_command_on_path(cxx_wrapper)) assert(mageec.is_command_on_path(fort_wrapper)) wrapper_flags = '' if debug: wrapper_flags += ' -fmageec-debug' wrapper_flags += ' -fmageec-mode=gather' wrapper_flags += ' -fmageec-database=' + database wrapper_flags += ' -fmageec-features=' + features wrapper_flags += ' -fmageec-out=' + compilations # add the wrapper flags to the front of the flags and then build using # the provided build script new_flags = wrapper_flags if build_flags != '': new_flags += ' ' + ' '.join(build_flags) res = mageec.build(src_dir=src_dir, build_dir=build_dir, install_dir=install_dir, build_script=build_script, cc=cc_wrapper, cxx=cxx_wrapper, fort=fort_wrapper, build_flags=new_flags) if not res: eprint ('-- Failed to build source \'' + src_dir + '\'') return 0 # Find the measure script. If it's not on the path, look in the # directory this script is in. If it's not in that directory then look # in the current working directory if mageec.is_command_on_path(measure_script): cmd_path = measure_script else: found_script = False if not os.path.isabs(measure_script): # Check if it's a script in the same directory as this script script_dir = os.path.dirname(os.path.realpath(__file__)) cmd_path = os.path.join(script_dir, measure_script) if os.path.exists(cmd_path): found_script = True if not found_script: cmd_path = os.path.abspath(measure_script) if not os.path.exists(cmd_path): eprint ('-- Failed to find script to measure benchmark result') return 0 # run the measurement script on each of the executables in the # install directory to produce a results file exec_files = [] exec_bits = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH for d in os.walk(install_dir): dir_path, _, files = d for f in files: f = os.path.join(dir_path, f) if os.path.isfile(f): stat_res = os.stat(f) stat_mode = stat_res.st_mode if stat_mode & exec_bits: exec_files.append(f) total = 0 for exec_path in exec_files: cmd = [cmd_path, '--exec-path', exec_path, '--compilation-ids', compilations, '--out', results_path] if exec_flags != '': cmd.append('--exec-flags') cmd.append(' '.join(exec_flags)) print (' '.join(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE) result_value = float(p.communicate()[0].decode().strip()) if p.returncode != 0: eprint ('-- Failed to measure executable \'' + exec_path + '\'') return 0 else: total += result_value return total
def main(): parser = argparse.ArgumentParser( description='Generate and build multiple versions of a source project') # required arguments parser.add_argument('--src-dir', nargs=1, required=True, help='Directory containing the source to build') parser.add_argument('--build-dir', nargs=1, required=True, help='Build directory') parser.add_argument('--install-dir', nargs=1, required=True, help='Install directory') parser.add_argument('--cc', nargs=1, required=True, help='Command to use to compile C source') parser.add_argument('--cxx', nargs=1, required=True, help='Command to use to compile C++ source') parser.add_argument('--fort', nargs=1, required=True, help='Command to use to compile Fortran source') parser.add_argument('--database', nargs=1, required=True, help='mageec database to store generated compilations in') parser.add_argument('--features', nargs=1, required=True, help='File containing extracted features for the source being built') parser.add_argument('--measure-script', nargs=1, required=True, help='Script to measure the resultant executables') parser.add_argument('--exec-flags', nargs=1, required=True, help='Flags to use when executing generated programs') # optional arguments parser.add_argument('--debug', action='store_true', required=False, help='Enable debug when doing feature extraction') parser.add_argument('--build-system', nargs=1, required=False, help='Build system to be used to build the source. May be \'cmake\', ' '\'configure\', or a script to be used to build the source') parser.add_argument('--flags', nargs=1, required=False, help='Common arguments to be used when building') parser.add_argument('--jobs', nargs=1, required=False, help='Number of jobs to run when building') parser.set_defaults(debug=False, build_system=[None], flags=[''], jobs=[1]) args = parser.parse_args(sys.argv[1:]) src_dir = os.path.abspath(args.src_dir[0]) build_dir = os.path.abspath(args.build_dir[0]) install_dir = os.path.abspath(args.install_dir[0]) cc = args.cc[0] cxx = args.cxx[0] fort = args.fort[0] database_path = os.path.abspath(args.database[0]) features_path = os.path.abspath(args.features[0]) measure_script = args.measure_script[0] exec_flags = args.exec_flags[0] if not os.path.exists(src_dir): print ('-- Source directory \'' + src_dir + '\' does not exist') return -1 if not os.path.exists(build_dir): os.makedirs(build_dir) if not os.path.exists(install_dir): os.makedirs(install_dir) if not os.path.exists(database_path): print ('-- Database \'' + database_path + '\' does not exist') return -1 if not os.path.exists(features_path): print ('-- Features file \'' + features_path + '\' does not exist') return -1 if not mageec.is_command_on_path(cc): print ('-- Compiler \'' + cc + '\' is not on the path') return -1 if not mageec.is_command_on_path(cxx): print ('-- Compiler \'' + cxx + '\' is not on the path') return -1 if not mageec.is_command_on_path(fort): print ('-- Compiler \'' + fort + '\' is not on the path') return -1 debug = args.debug build_system = args.build_system[0] flags = args.flags[0] jobs = int(args.jobs[0]) if jobs < 1: print ('-- Number of jobs must be a positive integer') return -1 res = combined_elimination(src_dir=src_dir, build_dir=build_dir, install_dir=install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=flags, jobs=jobs, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if not res: return -1 return 0
def combined_elimination(src_dir, build_dir, install_dir, build_system, cc, cxx, fort, flags, jobs, database_path, features_path, measure_script, exec_flags, debug): assert(os.path.exists(src_dir) and os.path.isabs(src_dir)) assert(os.path.exists(build_dir) and os.path.isabs(build_dir)) assert(os.path.exists(install_dir) and os.path.isabs(install_dir)) assert(os.path.exists(database_path)) assert(os.path.exists(features_path)) assert(mageec.is_command_on_path(cc)) assert(mageec.is_command_on_path(cxx)) assert(mageec.is_command_on_path(fort)) assert(jobs > 0) # Run at -O3 to get a point of comparison o3_flags = ['-O3'] run_flags = flags + ' ' + ' '.join(o3_flags) o3_build_dir = os.path.join(build_dir, 'o3') o3_install_dir = os.path.join(install_dir, 'o3') o3_res = build_and_measure(src_dir=src_dir, build_dir=o3_build_dir, install_dir=o3_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if o3_res <= 0: print ('-- O3 build failed. Exiting') return False # Also run at -Os to get a point of comparison os_flags = ['-Os'] run_flags = flags + ' ' + ' '.join(os_flags) os_build_dir = os.path.join(build_dir, 'os') os_install_dir = os.path.join(install_dir, 'os') os_res = build_and_measure(src_dir=src_dir, build_dir=os_build_dir, install_dir=os_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if os_res <= 0: print ('-- Os build failed. Exiting') return False run_id = 0 run_metadata = [] # Start with all flags enabled for the base line base_flags = list(all_flags) run_flags = flags + ' ' + ' '.join(base_flags) base_build_dir = os.path.join(build_dir, 'base-' + str(run_id)) base_install_dir = os.path.join(install_dir, 'base-' + str(run_id)) base_res = build_and_measure(src_dir=src_dir, build_dir=base_build_dir, install_dir=base_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if base_res <= 0: print ('-- Base build failed. Exiting') return False run_metadata.append(('base-' + str(run_id), run_flags, base_res)) run_id += 1 # Store the initial base run result for comparison later initial_base_res = base_res improvement = True flags_to_consider = list(all_flags) while improvement: improvement = False flags_with_improvement = [] # identify flags that bring improvement when disabled. # Each build is done in a seperate process test_pool = Pool(jobs) test_results = [] for flag in flags_to_consider: # do a run with each flag disabled in turn tmp_flags = list(base_flags) tmp_flags.remove(flag) tmp_flags = build_config(all_flags, tmp_flags) run_flags = flags + ' ' + ' '.join(tmp_flags) test_build_dir = os.path.join(build_dir, 'test-' + str(run_id)) test_install_dir = os.path.join(install_dir, 'test-' + str(run_id)) res = test_pool.apply_async(build_and_measure, (src_dir, test_build_dir, test_install_dir, build_system, cc, cxx, fort, run_flags, database_path, features_path, measure_script, exec_flags, debug)) test_results.append((run_id, run_flags, flag, res)) run_id += 1 # Wait for all of the test runs to complete, and then get their results for test_run_id, run_flags, flag, res in test_results: res.wait() for test_run_id, run_flags, flag, res in test_results: test_res = res.get() run_metadata.append(('test-' + str(test_run_id), run_flags, test_res)) if test_res <= 0: print ('-- Test run ' + str(test_run_id) + ' failed. Exiting') return False if test_res < base_res: flags_with_improvement += [(flag, test_res)] # Starting from the biggest improvement, check if disabling the # flag still brings improvement. If it does, then permanently # disable the flag, update the base flag configuration and remeasure flags_with_improvement = sorted(flags_with_improvement, key=lambda x: x[1]) for flag, result in flags_with_improvement: # do a run with each flag disabled in turn tmp_flags = list(base_flags) tmp_flags.remove(flag) tmp_flags = build_config(all_flags, tmp_flags) run_flags = flags + ' ' + ' '.join(tmp_flags) test_build_dir = os.path.join(build_dir, 'test-' + str(run_id)) test_install_dir = os.path.join(install_dir, 'test-' + str(run_id)) test_res = build_and_measure(src_dir=src_dir, build_dir=test_build_dir, install_dir=test_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if test_res <= 0: print ('-- Test run ' + str(run_id) + ' failed. Exiting') return False run_metadata.append(('test-' + str(run_id), run_flags, test_res)) run_id += 1 if test_res < base_res: # remove the flag permanently from the baseline build, and # remove it from further consideration base_flags.remove(flag) flags_to_consider.remove(flag) # build and measure the new baseline tmp_flags = build_config(all_flags, base_flags) run_flags = flags + ' ' + ' '.join(tmp_flags) base_build_dir = os.path.join(build_dir, 'base-' + str(run_id)) base_install_dir = os.path.join(install_dir, 'base-' + str(run_id)) base_res = build_and_measure(src_dir=src_dir, build_dir=base_build_dir, install_dir=base_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if base_res <= 0: print ('-- Base run ' + str(run_id) + ' failed. Exiting') return False run_metadata.append(('base-' + str(run_id), run_flags, base_res)) run_id += 1 improvement = True # Dump statistics now that we are no longer seeing any improvement for run in run_metadata: name = run[0] flags = run[1] res = run[2] base_improvement = float(initial_base_res) / res o3_improvement = float(o3_res) / res os_improvement = float(os_res) / res print (name + ',' + str(res) + ',' + str(base_improvement) + ',' + str(o3_improvement) + ',' + str(os_improvement) + ',' + flags)
def build_and_measure(src_dir, build_dir, install_dir, build_system, cc, cxx, fort, flags, database_path, features_path, measure_script, exec_flags, debug): if not os.path.exists(build_dir): os.makedirs(build_dir) if not os.path.exists(install_dir): os.makedirs(install_dir) # build the benchmark using the wrapper script # This will record the flags which the benchmark is being compiled with, # and associate these flags with previously extracted features through # a compilation id compilations_path = os.path.join(install_dir, 'compilations.csv') results_path = os.path.join(install_dir, 'results.csv') cc_wrapper = 'mageec-' + cc cxx_wrapper = 'mageec-' + cxx fort_wrapper = 'mageec-' + fort assert(mageec.is_command_on_path(cc_wrapper)) assert(mageec.is_command_on_path(cxx_wrapper)) assert(mageec.is_command_on_path(fort_wrapper)) wrapper_flags = "" if debug: wrapper_flags += ' -fmageec-debug' wrapper_flags += ' -fmageec-mode=gather' wrapper_flags += ' -fmageec-database=' + database_path wrapper_flags += ' -fmageec-features=' + features_path wrapper_flags += ' -fmageec-out=' + compilations_path # Add the wrapper flags to the front of the flags for the build new_flags = wrapper_flags + ' ' + flags res = mageec.build(src_dir=src_dir, build_dir=build_dir, install_dir=install_dir, build_system=build_system, cc=cc_wrapper, cxx=cxx_wrapper, fort=fort_wrapper, flags=new_flags) # Run the measurement script to produce a results file which can be # loaded into the database cmd_path = measure_script if not os.path.isabs(cmd_path): cmd_path = os.path.join(os.getcwd(), cmd_path) if os.path.exists(cmd_path): cmd_name = cmd_path elif mageec.is_command_on_path(measure_script): cmd_name = measure_script else: print ('Failed to find script to measure benchmark result') return 0 cmd = [cmd_name, '--install-dir', install_dir, '--flags', exec_flags, '--compilation-ids', compilations_path, '--out', results_path] ret = subprocess.call(cmd) if ret != 0: return 0 # Read in the results file, we want to extract the results for all of the # source files which make up the executable total = 0 result_files = set() for line in open(results_path): values = line.split(',') # Ignore anything which isn't a 'result' line if len(values) != 7: continue if values[3] != 'result': continue # Only accumulate results for modules if values[1] != 'module': continue src_file = values[0] module_name = values[2] result = float(values[6]) if src_file in result_files: print ('-- Duplicate results for module ' + module_name) return 0 result_files.add(src_file) # accumulate the measured results for each of the modules in the # program. This total will be used to drive the next stage of # combined elimination # # FIXME: This isn't ideal, as improvements in some modules could be # masked by regressions in other modules. Ideally we would do combined # elimination on each module individually, but that would be # prohibitively expensive. total += result # FIXME: Actually adding the results to the database is currently left as an # exercise for the user return total
def main(): parser = argparse.ArgumentParser( description='Generate and build multiple versions of a source project') # required arguments parser.add_argument('--src-dir', nargs=1, required=True, help='Directory containing the source to build') parser.add_argument('--build-dir', nargs=1, required=True, help='Build directory') parser.add_argument('--install-dir', nargs=1, required=True, help='Install directory') parser.add_argument('--cc', nargs=1, required=True, help='Command to use to compile C source') parser.add_argument('--cxx', nargs=1, required=True, help='Command to use to compile C++ source') parser.add_argument('--fort', nargs=1, required=True, help='Command to use to compile Fortran source') parser.add_argument( '--database', nargs=1, required=True, help='mageec database to store generated compilations in') parser.add_argument( '--features', nargs=1, required=True, help='File containing extracted features for the source being built') parser.add_argument('--measure-script', nargs=1, required=True, help='Script to measure the resultant executables') parser.add_argument('--exec-flags', nargs=1, required=True, help='Flags to use when executing generated programs') # optional arguments parser.add_argument('--debug', action='store_true', required=False, help='Enable debug when doing feature extraction') parser.add_argument( '--build-system', nargs=1, required=False, help='Build system to be used to build the source. May be \'cmake\', ' '\'configure\', or a script to be used to build the source') parser.add_argument('--flags', nargs=1, required=False, help='Common arguments to be used when building') parser.add_argument('--jobs', nargs=1, required=False, help='Number of jobs to run when building') parser.set_defaults(debug=False, build_system=[None], flags=[''], jobs=[1]) args = parser.parse_args(sys.argv[1:]) src_dir = os.path.abspath(args.src_dir[0]) build_dir = os.path.abspath(args.build_dir[0]) install_dir = os.path.abspath(args.install_dir[0]) cc = args.cc[0] cxx = args.cxx[0] fort = args.fort[0] database_path = os.path.abspath(args.database[0]) features_path = os.path.abspath(args.features[0]) measure_script = args.measure_script[0] exec_flags = args.exec_flags[0] if not os.path.exists(src_dir): print('-- Source directory \'' + src_dir + '\' does not exist') return -1 if not os.path.exists(build_dir): os.makedirs(build_dir) if not os.path.exists(install_dir): os.makedirs(install_dir) if not os.path.exists(database_path): print('-- Database \'' + database_path + '\' does not exist') return -1 if not os.path.exists(features_path): print('-- Features file \'' + features_path + '\' does not exist') return -1 if not mageec.is_command_on_path(cc): print('-- Compiler \'' + cc + '\' is not on the path') return -1 if not mageec.is_command_on_path(cxx): print('-- Compiler \'' + cxx + '\' is not on the path') return -1 if not mageec.is_command_on_path(fort): print('-- Compiler \'' + fort + '\' is not on the path') return -1 debug = args.debug build_system = args.build_system[0] flags = args.flags[0] jobs = int(args.jobs[0]) if jobs < 1: print('-- Number of jobs must be a positive integer') return -1 res = combined_elimination(src_dir=src_dir, build_dir=build_dir, install_dir=install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=flags, jobs=jobs, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if not res: return -1 return 0
def combined_elimination(src_dir, build_dir, install_dir, build_system, cc, cxx, fort, flags, jobs, database_path, features_path, measure_script, exec_flags, debug): assert (os.path.exists(src_dir) and os.path.isabs(src_dir)) assert (os.path.exists(build_dir) and os.path.isabs(build_dir)) assert (os.path.exists(install_dir) and os.path.isabs(install_dir)) assert (os.path.exists(database_path)) assert (os.path.exists(features_path)) assert (mageec.is_command_on_path(cc)) assert (mageec.is_command_on_path(cxx)) assert (mageec.is_command_on_path(fort)) assert (jobs > 0) # Run at -O3 to get a point of comparison o3_flags = ['-O3'] run_flags = flags + ' ' + ' '.join(o3_flags) o3_build_dir = os.path.join(build_dir, 'o3') o3_install_dir = os.path.join(install_dir, 'o3') o3_res = build_and_measure(src_dir=src_dir, build_dir=o3_build_dir, install_dir=o3_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if o3_res <= 0: print('-- O3 build failed. Exiting') return False # Also run at -Os to get a point of comparison os_flags = ['-Os'] run_flags = flags + ' ' + ' '.join(os_flags) os_build_dir = os.path.join(build_dir, 'os') os_install_dir = os.path.join(install_dir, 'os') os_res = build_and_measure(src_dir=src_dir, build_dir=os_build_dir, install_dir=os_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if os_res <= 0: print('-- Os build failed. Exiting') return False run_id = 0 run_metadata = [] # Start with all flags enabled for the base line base_flags = list(all_flags) run_flags = flags + ' ' + ' '.join(base_flags) base_build_dir = os.path.join(build_dir, 'base-' + str(run_id)) base_install_dir = os.path.join(install_dir, 'base-' + str(run_id)) base_res = build_and_measure(src_dir=src_dir, build_dir=base_build_dir, install_dir=base_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if base_res <= 0: print('-- Base build failed. Exiting') return False run_metadata.append(('base-' + str(run_id), run_flags, base_res)) run_id += 1 # Store the initial base run result for comparison later initial_base_res = base_res improvement = True flags_to_consider = list(all_flags) while improvement: improvement = False flags_with_improvement = [] # identify flags that bring improvement when disabled. # Each build is done in a seperate process test_pool = Pool(jobs) test_results = [] for flag in flags_to_consider: # do a run with each flag disabled in turn tmp_flags = list(base_flags) tmp_flags.remove(flag) tmp_flags = build_config(all_flags, tmp_flags) run_flags = flags + ' ' + ' '.join(tmp_flags) test_build_dir = os.path.join(build_dir, 'test-' + str(run_id)) test_install_dir = os.path.join(install_dir, 'test-' + str(run_id)) res = test_pool.apply_async( build_and_measure, (src_dir, test_build_dir, test_install_dir, build_system, cc, cxx, fort, run_flags, database_path, features_path, measure_script, exec_flags, debug)) test_results.append((run_id, run_flags, flag, res)) run_id += 1 # Wait for all of the test runs to complete, and then get their results for test_run_id, run_flags, flag, res in test_results: res.wait() for test_run_id, run_flags, flag, res in test_results: test_res = res.get() run_metadata.append( ('test-' + str(test_run_id), run_flags, test_res)) if test_res <= 0: print('-- Test run ' + str(test_run_id) + ' failed. Exiting') return False if test_res < base_res: flags_with_improvement += [(flag, test_res)] # Starting from the biggest improvement, check if disabling the # flag still brings improvement. If it does, then permanently # disable the flag, update the base flag configuration and remeasure flags_with_improvement = sorted(flags_with_improvement, key=lambda x: x[1]) for flag, result in flags_with_improvement: # do a run with each flag disabled in turn tmp_flags = list(base_flags) tmp_flags.remove(flag) tmp_flags = build_config(all_flags, tmp_flags) run_flags = flags + ' ' + ' '.join(tmp_flags) test_build_dir = os.path.join(build_dir, 'test-' + str(run_id)) test_install_dir = os.path.join(install_dir, 'test-' + str(run_id)) test_res = build_and_measure(src_dir=src_dir, build_dir=test_build_dir, install_dir=test_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if test_res <= 0: print('-- Test run ' + str(run_id) + ' failed. Exiting') return False run_metadata.append(('test-' + str(run_id), run_flags, test_res)) run_id += 1 if test_res < base_res: # remove the flag permanently from the baseline build, and # remove it from further consideration base_flags.remove(flag) flags_to_consider.remove(flag) # build and measure the new baseline tmp_flags = build_config(all_flags, base_flags) run_flags = flags + ' ' + ' '.join(tmp_flags) base_build_dir = os.path.join(build_dir, 'base-' + str(run_id)) base_install_dir = os.path.join(install_dir, 'base-' + str(run_id)) base_res = build_and_measure(src_dir=src_dir, build_dir=base_build_dir, install_dir=base_install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=run_flags, database_path=database_path, features_path=features_path, measure_script=measure_script, exec_flags=exec_flags, debug=debug) if base_res <= 0: print('-- Base run ' + str(run_id) + ' failed. Exiting') return False run_metadata.append( ('base-' + str(run_id), run_flags, base_res)) run_id += 1 improvement = True # Dump statistics now that we are no longer seeing any improvement for run in run_metadata: name = run[0] flags = run[1] res = run[2] base_improvement = float(initial_base_res) / res o3_improvement = float(o3_res) / res os_improvement = float(os_res) / res print(name + ',' + str(res) + ',' + str(base_improvement) + ',' + str(o3_improvement) + ',' + str(os_improvement) + ',' + flags)
def build_and_measure(src_dir, build_dir, install_dir, build_system, cc, cxx, fort, flags, database_path, features_path, measure_script, exec_flags, debug): if not os.path.exists(build_dir): os.makedirs(build_dir) if not os.path.exists(install_dir): os.makedirs(install_dir) # build the benchmark using the wrapper script # This will record the flags which the benchmark is being compiled with, # and associate these flags with previously extracted features through # a compilation id compilations_path = os.path.join(install_dir, 'compilations.csv') results_path = os.path.join(install_dir, 'results.csv') cc_wrapper = 'mageec-' + cc cxx_wrapper = 'mageec-' + cxx fort_wrapper = 'mageec-' + fort assert (mageec.is_command_on_path(cc_wrapper)) assert (mageec.is_command_on_path(cxx_wrapper)) assert (mageec.is_command_on_path(fort_wrapper)) wrapper_flags = "" if debug: wrapper_flags += ' -fmageec-debug' wrapper_flags += ' -fmageec-mode=gather' wrapper_flags += ' -fmageec-database=' + database_path wrapper_flags += ' -fmageec-features=' + features_path wrapper_flags += ' -fmageec-out=' + compilations_path # Add the wrapper flags to the front of the flags for the build new_flags = wrapper_flags + ' ' + flags res = mageec.build(src_dir=src_dir, build_dir=build_dir, install_dir=install_dir, build_system=build_system, cc=cc_wrapper, cxx=cxx_wrapper, fort=fort_wrapper, flags=new_flags) # Run the measurement script to produce a results file which can be # loaded into the database cmd_path = measure_script if not os.path.isabs(cmd_path): cmd_path = os.path.join(os.getcwd(), cmd_path) if os.path.exists(cmd_path): cmd_name = cmd_path elif mageec.is_command_on_path(measure_script): cmd_name = measure_script else: print('Failed to find script to measure benchmark result') return 0 cmd = [ cmd_name, '--install-dir', install_dir, '--flags', exec_flags, '--compilation-ids', compilations_path, '--out', results_path ] ret = subprocess.call(cmd) if ret != 0: return 0 # Read in the results file, we want to extract the results for all of the # source files which make up the executable total = 0 result_files = set() for line in open(results_path): values = line.split(',') # Ignore anything which isn't a 'result' line if len(values) != 7: continue if values[3] != 'result': continue # Only accumulate results for modules if values[1] != 'module': continue src_file = values[0] module_name = values[2] result = float(values[6]) if src_file in result_files: print('-- Duplicate results for module ' + module_name) return 0 result_files.add(src_file) # accumulate the measured results for each of the modules in the # program. This total will be used to drive the next stage of # combined elimination # # FIXME: This isn't ideal, as improvements in some modules could be # masked by regressions in other modules. Ideally we would do combined # elimination on each module individually, but that would be # prohibitively expensive. total += result # FIXME: Actually adding the results to the database is currently left as an # exercise for the user return total
def main(): parser = argparse.ArgumentParser(description='Perform feature extraction') # required arguments parser.add_argument('--src-dir', nargs=1, required=True, help='Directory containing the source to build') parser.add_argument('--build-dir', nargs=1, required=True, help='Build directory') parser.add_argument('--install-dir', nargs=1, required=True, help='Install directory') parser.add_argument('--cc', nargs=1, required=True, help='Command to use to compile C source') parser.add_argument('--cxx', nargs=1, required=True, help='Command to use to compile C++ source') parser.add_argument('--fort', nargs=1, required=True, help='Command to use to compile Fortan source') parser.add_argument( '--database', nargs=1, required=True, help='mageec database to store extracted features into') parser.add_argument( '--mageec-library-path', nargs=1, required=True, help='Path to the directory holding the mageec libraries') parser.add_argument('--out', nargs=1, required=True, help='File to output the extracted features to') # optional arguments parser.add_argument('--debug', action='store_true', required=False, help='Enable debug when doing feature extraction') parser.add_argument( '--build-system', nargs=1, required=False, help='Build system to be used to build the source. May be \'cmake\', ' '\'configure\', or a script to be used to build the source') parser.add_argument('--flags', nargs=1, required=False, help='Common arguments to be used when building') parser.set_defaults(debug=False, build_system=[None], flags=['']) args = parser.parse_args(sys.argv[1:]) src_dir = os.path.abspath(args.src_dir[0]) build_dir = os.path.abspath(args.build_dir[0]) install_dir = os.path.abspath(args.install_dir[0]) cc = args.cc[0] cxx = args.cxx[0] fort = args.fort[0] database_path = os.path.abspath(args.database[0]) mageec_lib_path = os.path.abspath(args.mageec_library_path[0]) out_path = os.path.abspath(args.out[0]) debug = args.debug build_system = args.build_system[0] flags = args.flags[0] # TODO: Extension should depend on platform gcc_plugin_name = mageec.get_gcc_plugin_name() + '.so' if not os.path.exists(src_dir): print('-- Source directory \'' + src_dir + '\' does not exist') return -1 if not os.path.exists(build_dir): os.makedirs(build_dir) if not os.path.exists(install_dir): os.makedirs(install_dir) if not os.path.exists(database_path): print('-- Database \'' + database_path + '\' does not exist') return -1 if not mageec.is_command_on_path(cc): print('-- Compiler \'' + cc + '\' is not on the path') return -1 print('-- Checking for mageec plugin \'' + gcc_plugin_name + '\'') gcc_plugin_path = os.path.join(mageec_lib_path, gcc_plugin_name) if not os.path.exists(gcc_plugin_path): print('-- Could not find gcc plugin') return -1 build_dir = os.path.join(build_dir, 'feature_extract') install_dir = os.path.join(install_dir, 'feature_extract') if not os.path.exists(build_dir): os.makedirs(build_dir) if not os.path.exists(install_dir): os.makedirs(install_dir) res = mageec.feature_extract(src_dir=src_dir, build_dir=build_dir, install_dir=install_dir, build_system=build_system, cc=cc, cxx=cxx, fort=fort, flags=flags, database_path=database_path, gcc_plugin_path=gcc_plugin_path, debug=debug, out_path=out_path) if not res: return -1 return 0