def extract_data(subset, pickle_symfiles=False):
try:
# Unpack all the seeds we need and create the relative path
relpath = support.relpath_for_seeds(*subset)
print('************ Extracting for path ' + relpath + ' **********')
for (benchmark, name) in support.benchmarks_gen():
# Create the output directory for this benchmark
build_dir = os.path.join(config.build_dir, relpath, benchmark)
data_dir = os.path.join(config.data_dir, relpath, benchmark)
os.makedirs(data_dir)
os.symlink(build_dir, os.path.join(data_dir, 'build'))
with open(os.path.join(data_dir, 'symfile'), 'w') as f_sym:
# Extract the actual symfile using dump_syms. This tool creates a LOT of warnings so we redirect stderr to /dev/null
subprocess.check_call([config.dump_syms, os.path.join(build_dir, name)], stdout=f_sym, stderr=subprocess.DEVNULL)
# If we're dealing with the base we have to do some more stuff
if not subset:
# For every protection, copy over the opportunity log, if any
for opportunity_log in [s.opportunity_log for s in seed.get_types() if s.opportunity_log]:
shutil.copy2(os.path.join(support.create_path_for_seeds(config.build_dir), benchmark, opportunity_log), data_dir)
# Copy over the linker map
shutil.copy2(os.path.join(build_dir, name + '.map'), os.path.join(data_dir, 'map'))
# Extract the section alignment information
linker.gather_section_alignment(os.path.join(build_dir, name + '.map'), os.path.join(data_dir, 'sections'))
if pickle_symfiles:
# Get the symfile
symfile_path = os.path.join(os.path.join(data_dir, 'symfile'))
symfile = SymFile().read_f(symfile_path)
# Pickle it
with open(os.path.join(data_dir, 'pickled_symfile'), 'wb') as f_pickle:
pickle.dump(symfile, f_pickle)
if not subset:
data_dir = os.path.join(config.data_dir, relpath)
# For every protection, copy over the opportunity logs for the extra build archives/objects (which are the same for every benchmark), if any.
# Also copy over the build_prefix (in symlink form).
for a in os.listdir(support.create_path_for_seeds(config.extra_build_dir)):
a_path = os.path.join(support.create_path_for_seeds(config.extra_build_dir), a)
os.symlink(os.readlink(os.path.join(a_path, 'build')), os.path.join(data_dir, 'build.' + a))
for opportunity_log in [s.opportunity_log for s in seed.get_types() if s.opportunity_log]:
shutil.copy2(os.path.join(a_path, opportunity_log), os.path.join(data_dir, opportunity_log + '.' + a))
except Exception:
logging.getLogger().exception('Data extraction failed for ' + relpath)
def seed_tuple(s):
tokens = s.split(',')
types = seed.get_types()
assert len(tokens) == len(
types
), 'The number of seeds on the command line differ from the number of protections.'
# First convert the tokens to integers, then create the actual seeds
int_seeds = [int(token) if token else 0 for token in tokens]
return [cls(s) for cls, s in zip(types, int_seeds)]
def main():
# Create the report
print('************ Creating report on delta data sizes **********')
sheets = {}
for subset in support.subsets_gen(seed.get_types(), False):
# Create the sheet for this subset and put it in the dictionary
name = ','.join(
[t.__name__ for t in subset]
) # Create the sheet name out of the typenames of the seeds in the subset
sheet = pyexcel.Sheet(name=name)
sheets[name] = sheet
# Create the first few columns. The first is for the benchmarks, second is average, and third is max (to be filled in later).
rownames = [benchmark for benchmark, _ in support.benchmarks_gen()]
sheet.column += ['Delta data'] + rownames
sheet.column += ['AVG'] + [''] * len(rownames)
sheet.column += ['MAX'] + [''] * len(rownames)
for seed_tuple in support.seeds_gen(*subset):
# Empty cell
sizes = ['']
# Get all the sizes of the patches
for benchmark, _ in support.benchmarks_gen():
dd = os.path.join(
support.create_path_for_seeds(config.patches_dir,
*seed_tuple), benchmark,
'delta_data')
if os.path.exists(dd):
sizes.append(os.stat(dd).st_size)
else:
sizes.append('FAIL')
sheet.column += sizes
# Calculate in the average and the max
for row in list(sheet.rows())[1:]:
sizes = [elem for elem in row[3:] if isinstance(elem, int)]
if sizes:
row[1] = sum(sizes) // len(sizes)
row[2] = max(sizes)
# Create the report book and write it out
report = pyexcel.Book(sheets=sheets)
report.save_as(os.path.join(config.reports_dir, 'delta_data_sizes.ods'))
def main(compile_args=[]):
# Use the default template linker script to minimize the differences when we start protecting at link time
linker.create_linker_script(None)
# Clean up from possible previous runs
shutil.rmtree(config.build_dir, True)
shutil.rmtree(config.extra_build_dir, True)
os.mkdir(config.extra_build_dir)
# Start by compiling for the default binaries. Build up the compile options, starting from the binary options, adding the default
# compile options for the diferent protections, then the compile arguments passed on the command line.
print('************ Building default binaries... **********')
default_compile_options = get_default_compile_options()
compile_options = default_compile_options + compile_args
# We start building the extra binaries, and add their extra compile options to the ones we use to build SPEC.
compile_options += build_extra(
support.create_path_for_seeds(config.extra_build_dir), compile_options)
build_spec(support.create_path_for_seeds(config.build_dir),
' '.join(compile_options))
print('************ Build finished. **********')
# Next we compile the protected binaries. We build up the compile options, starting from the binary options, adding the
# compile options for the different protections (based on the associated seed), then the compile arguments passed on the
# command line.
for protections in support.build_subsets_gen(seed.get_types(), False):
for seeds in support.seeds_gen(*protections):
print('************ Building protected binary for ' +
' '.join([repr(s) for s in seeds]) + ' ... **********')
compile_options = list(default_compile_options)
for s in seeds:
compile_options += s.diversify_build()
compile_options += compile_args
# We start building the extra binaries, and add their extra compile options to the ones we use to build SPEC.
compile_options += build_extra(
support.create_path_for_seeds(config.extra_build_dir, *seeds),
compile_options)
build_spec(support.create_path_for_seeds(config.build_dir, *seeds),
' '.join(compile_options))
print('************ Build finished. **********')
def all_seeds_gen():
for s in seeds_gen(*seed.get_types()):
yield s
def main():
# We prepare by copying the binaries to the location where they'll be protected
print(
'************ Preparing for link-time protections by copying binaries **********'
)
for link_protections in support.link_subsets_gen(seed.get_types(), False):
for build_protections in support.build_subsets_gen(seed.get_types()):
for build_seeds, link_seeds in zip(
support.seeds_gen(*build_protections),
support.seeds_gen(*link_protections)):
support.copy_spec_tree(
support.create_path_for_seeds(config.build_dir,
*build_seeds),
support.create_path_for_seeds(config.build_dir,
*build_seeds, *link_seeds))
for (benchmark, name) in support.benchmarks_gen():
# Get all the sections from all the objects in the build directory
print('************************* ' + benchmark +
' **********************')
print(
'************************* Gathering sections **********************'
)
linkermap = Map(
os.path.join(support.create_path_for_seeds(config.build_dir),
benchmark, name + '.map'))
# Get all the pre_sections and make linker rules out of them. We do this so that they can't change order (which apparently, they can...).
# Use a '*' (even though it is unnecessary) so Diablo's map parser will recognize it as a pattern.
# Remove the name of the encompassing archive from an object (if it exists), else Diablo can't handle this
pre_sections = [[
support.get_objname(section.obj) + '*(' + section.name + ')'
] for section in linkermap.pre_sections]
# We want to create a list of all linker rules that can be altered. Ideally we would simply take all sections currently in the
# linkermap (that we want to change) and convert them into a rule that matches only that section from its specific object.
# Unfortunately the linker is a bit fickle in its handling of weak symbols. If N sections (coming from N different objects)
# exist that define the same symbol, the linker will select only one (from one object) to place in the binary and discard
# the rest. The problem is that during the second, protected link (using the linker script we generate here) the
# linker won't necessarily select a weak symbol section from the same object as it did in the first link. The custom rule
# (which includes the name of the object the section came from the first link) won't match and, for example, the section would
# be put after the sections that ARE shuffled. To avoid this, we also keep all discarded sections, and then create N rules for the
# section (one for each object). These rules stay together during the protections, thus guaranteeing the right location of the section.
main_sections = []
for section in linkermap.main_sections:
rule_list = []
# Create the linker rules and insert them in the list
# Use a '*' (even though it is unnecessary) so Diablo's map parser will recognize it as a pattern.
# Remove the name of the encompassing archive from an object (if it exists), else Diablo can't handle this
suffix = '*(' + section.name + ')'
rule_list.append(support.get_objname(section.obj) + suffix)
for discarded in linkermap.discarded_sections:
if discarded.name == section.name:
rule_list.append(
support.get_objname(discarded.obj) + suffix)
# Add the rule list to the list of lists
main_sections.append(rule_list)
# Perform the actual link-time protections by creating a new linker script (in which sections can change order) and relinking
for link_protections in support.link_subsets_gen(
seed.get_types(), False):
# First create a new linker script for every combination of link protections
for link_seeds in support.seeds_gen(*link_protections):
print('************ Protecting binary at link level for ' +
' '.join([repr(s)
for s in link_seeds]) + ' ... **********')
# Copy the list so that every time we start from the original array
protected_pre_sections = list(pre_sections)
protected_main_sections = list(main_sections)
for s in link_seeds:
protected_pre_sections, protected_main_sections = s.diversify_link(
protected_pre_sections, protected_main_sections)
# Create diversified link script
linker.create_linker_script(
protected_pre_sections + protected_main_sections,
os.path.join(
support.create_path_for_seeds(config.build_dir,
*link_seeds), benchmark,
'link.xc'))
for build_protections in support.build_subsets_gen(
seed.get_types()):
for build_seeds, link_seeds in zip(
support.seeds_gen(*build_protections),
support.seeds_gen(*link_protections)):
# Get the link command, then adapt it to use our new linker script
directory = os.path.join(
support.create_path_for_seeds(config.build_dir,
*build_seeds,
*link_seeds), benchmark)
with open(os.path.join(directory, 'make.out'), 'r') as f:
cmd = list(f)[-1].rstrip()
new_script = os.path.join(
support.create_path_for_seeds(config.build_dir,
*link_seeds), benchmark,
'link.xc')
cmd = cmd.replace(config.link_script, new_script)
# Execute them with our diversified linker script
subprocess.check_call(shlex.split(cmd), cwd=directory)
def get_default_compile_options(with_protections=True):
default_compile_options = [config.binary_options]
if with_protections:
for protection in seed.get_types():
default_compile_options += protection.default_compile_options
return default_compile_options