def inject_data(benchmark, name, seeds, encrypt):
try:
# Determine all paths
binary = os.path.join(
support.create_path_for_seeds(config.build_dir, *seeds), benchmark,
name)
patch_dir = os.path.join(
support.create_path_for_seeds(config.patches_dir, *seeds),
benchmark)
print('************ Injecting delta data for binary ' + binary +
' **********')
# Get the patch
with open(os.path.join(patch_dir, 'patch'), 'rb') as f_p:
patch = f_p.read()
# Encode the delta data and write it to a file
dd = delta_data.encode(seeds, patch, encrypt)
dd_path = os.path.join(patch_dir, 'delta_data')
with open(dd_path, 'wb') as f_dd:
f_dd.write(dd)
# Do the actual injection
delta_data.inject(binary, dd_path)
except Exception:
logging.getLogger().exception('Delta data injection failed for ' +
binary)
def extract_data(subset, pickle_symfiles=False):
try:
# Unpack all the seeds we need and create the relative path
relpath = support.relpath_for_seeds(*subset)
print('************ Extracting for path ' + relpath + ' **********')
for (benchmark, name) in support.benchmarks_gen():
# Create the output directory for this benchmark
build_dir = os.path.join(config.build_dir, relpath, benchmark)
data_dir = os.path.join(config.data_dir, relpath, benchmark)
os.makedirs(data_dir)
os.symlink(build_dir, os.path.join(data_dir, 'build'))
with open(os.path.join(data_dir, 'symfile'), 'w') as f_sym:
# Extract the actual symfile using dump_syms. This tool creates a LOT of warnings so we redirect stderr to /dev/null
subprocess.check_call([config.dump_syms, os.path.join(build_dir, name)], stdout=f_sym, stderr=subprocess.DEVNULL)
# If we're dealing with the base we have to do some more stuff
if not subset:
# For every protection, copy over the opportunity log, if any
for opportunity_log in [s.opportunity_log for s in seed.get_types() if s.opportunity_log]:
shutil.copy2(os.path.join(support.create_path_for_seeds(config.build_dir), benchmark, opportunity_log), data_dir)
# Copy over the linker map
shutil.copy2(os.path.join(build_dir, name + '.map'), os.path.join(data_dir, 'map'))
# Extract the section alignment information
linker.gather_section_alignment(os.path.join(build_dir, name + '.map'), os.path.join(data_dir, 'sections'))
if pickle_symfiles:
# Get the symfile
symfile_path = os.path.join(os.path.join(data_dir, 'symfile'))
symfile = SymFile().read_f(symfile_path)
# Pickle it
with open(os.path.join(data_dir, 'pickled_symfile'), 'wb') as f_pickle:
pickle.dump(symfile, f_pickle)
if not subset:
data_dir = os.path.join(config.data_dir, relpath)
# For every protection, copy over the opportunity logs for the extra build archives/objects (which are the same for every benchmark), if any.
# Also copy over the build_prefix (in symlink form).
for a in os.listdir(support.create_path_for_seeds(config.extra_build_dir)):
a_path = os.path.join(support.create_path_for_seeds(config.extra_build_dir), a)
os.symlink(os.readlink(os.path.join(a_path, 'build')), os.path.join(data_dir, 'build.' + a))
for opportunity_log in [s.opportunity_log for s in seed.get_types() if s.opportunity_log]:
shutil.copy2(os.path.join(a_path, opportunity_log), os.path.join(data_dir, opportunity_log + '.' + a))
except Exception:
logging.getLogger().exception('Data extraction failed for ' + relpath)
def main():
# Create the sheet
print('************ Creating report on binary sizes **********')
sheet = pyexcel.Sheet()
rownames = [benchmark for benchmark, _ in support.benchmarks_gen()]
sheet.column += ['Binary default'] + rownames + ['Binary diversified'
] + rownames
# Get all the sizes of the default binaries
sizes = ['']
for (benchmark, name) in support.benchmarks_gen():
binary = os.path.join(support.create_path_for_seeds(config.build_dir),
benchmark, name)
if os.path.exists(binary):
sizes.append(get_binary_stripped_size(binary))
else:
sizes.append('FAIL')
# Create the AVG and MAX columns
sheet.column += sizes + ['AVG'] + [''] * len(rownames)
sheet.column += sizes + ['MAX'] + [''] * len(rownames)
for seeds in support.all_seeds_gen():
# Get all the sizes of the diversified binaries
sizes = [''] * (len(rownames) + 2)
for (benchmark, name) in support.benchmarks_gen():
binary = os.path.join(
support.create_path_for_seeds(config.build_dir, *seeds),
benchmark, name)
if os.path.exists(binary):
sizes.append(get_binary_stripped_size(binary))
else:
sizes.append('FAIL')
sheet.column += sizes
# Calculate in the average and the max
for row in (row for row in sheet.rows()
if not row[0].startswith('Binary')):
sizes = [elem for elem in row[3:] if isinstance(elem, int)]
if sizes:
row[1] = sum(sizes) // len(sizes)
row[2] = max(sizes)
# Create the report book and write it out
report = pyexcel.Book(sheets={'Sizes': sheet})
report.save_as(os.path.join(config.reports_dir, 'binary_sizes.ods'))
def create_patch(benchmark, seeds):
try:
# Determine all paths
base_data = os.path.join(support.create_path_for_seeds(config.data_dir), benchmark)
div_symfile_path = os.path.join(support.create_path_for_seeds(config.data_dir, *seeds), benchmark, 'symfile')
patch_dir = os.path.join(support.create_path_for_seeds(config.patches_dir, *seeds), benchmark)
print('************ Creating patch for benchmark ' + patch_dir + ' **********')
# Make the subdirectories and create the patch
os.makedirs(patch_dir)
patch.patch(base_data, seeds, div_symfile_path=div_symfile_path, output_dir=patch_dir)
# Verify the patch is correct
if not patch.patch(base_data, seeds, div_symfile_path=div_symfile_path, patch_path=os.path.join(patch_dir, 'patch')):
os.remove(os.path.join(patch_dir, 'patch'))
logging.getLogger().error('Patch verification failed for ' + patch_dir)
except Exception:
logging.getLogger().exception('Patch creation failed for ' + patch_dir)
def main(compile_args=[]):
# Use the default template linker script to minimize the differences when we start protecting at link time
linker.create_linker_script(None)
# Clean up from possible previous runs
shutil.rmtree(config.build_dir, True)
shutil.rmtree(config.extra_build_dir, True)
os.mkdir(config.extra_build_dir)
# Start by compiling for the default binaries. Build up the compile options, starting from the binary options, adding the default
# compile options for the diferent protections, then the compile arguments passed on the command line.
print('************ Building default binaries... **********')
default_compile_options = get_default_compile_options()
compile_options = default_compile_options + compile_args
# We start building the extra binaries, and add their extra compile options to the ones we use to build SPEC.
compile_options += build_extra(
support.create_path_for_seeds(config.extra_build_dir), compile_options)
build_spec(support.create_path_for_seeds(config.build_dir),
' '.join(compile_options))
print('************ Build finished. **********')
# Next we compile the protected binaries. We build up the compile options, starting from the binary options, adding the
# compile options for the different protections (based on the associated seed), then the compile arguments passed on the
# command line.
for protections in support.build_subsets_gen(seed.get_types(), False):
for seeds in support.seeds_gen(*protections):
print('************ Building protected binary for ' +
' '.join([repr(s) for s in seeds]) + ' ... **********')
compile_options = list(default_compile_options)
for s in seeds:
compile_options += s.diversify_build()
compile_options += compile_args
# We start building the extra binaries, and add their extra compile options to the ones we use to build SPEC.
compile_options += build_extra(
support.create_path_for_seeds(config.extra_build_dir, *seeds),
compile_options)
build_spec(support.create_path_for_seeds(config.build_dir, *seeds),
' '.join(compile_options))
print('************ Build finished. **********')
def main():
# Create the report
print('************ Creating report on delta data sizes **********')
sheets = {}
for subset in support.subsets_gen(seed.get_types(), False):
# Create the sheet for this subset and put it in the dictionary
name = ','.join(
[t.__name__ for t in subset]
) # Create the sheet name out of the typenames of the seeds in the subset
sheet = pyexcel.Sheet(name=name)
sheets[name] = sheet
# Create the first few columns. The first is for the benchmarks, second is average, and third is max (to be filled in later).
rownames = [benchmark for benchmark, _ in support.benchmarks_gen()]
sheet.column += ['Delta data'] + rownames
sheet.column += ['AVG'] + [''] * len(rownames)
sheet.column += ['MAX'] + [''] * len(rownames)
for seed_tuple in support.seeds_gen(*subset):
# Empty cell
sizes = ['']
# Get all the sizes of the patches
for benchmark, _ in support.benchmarks_gen():
dd = os.path.join(
support.create_path_for_seeds(config.patches_dir,
*seed_tuple), benchmark,
'delta_data')
if os.path.exists(dd):
sizes.append(os.stat(dd).st_size)
else:
sizes.append('FAIL')
sheet.column += sizes
# Calculate in the average and the max
for row in list(sheet.rows())[1:]:
sizes = [elem for elem in row[3:] if isinstance(elem, int)]
if sizes:
row[1] = sum(sizes) // len(sizes)
row[2] = max(sizes)
# Create the report book and write it out
report = pyexcel.Book(sheets=sheets)
report.save_as(os.path.join(config.reports_dir, 'delta_data_sizes.ods'))
def main():
# Create the sheet
print('************ Creating report on opportunity log sizes **********')
sheet = pyexcel.Sheet()
rownames = [benchmark for benchmark, _ in support.benchmarks_gen()]
sheet.column += ['Opportunity log'] + rownames
# Get all the sizes of the opportunity logs
sizes = ['']
for (benchmark, name) in support.benchmarks_gen():
data_dir = os.path.join(support.create_path_for_seeds(config.data_dir),
benchmark)
sizes.append(get_opportunity_log_size(data_dir))
sheet.column += sizes
# Create the report book and write it out
report = pyexcel.Book(sheets={'Sizes': sheet})
report.save_as(
os.path.join(config.reports_dir, 'opportunity_log_sizes.ods'))
def main():
# Create the sheet
print('************ Creating report on patch timing **********')
sheet = pyexcel.Sheet()
rownames = [benchmark for benchmark, _ in support.benchmarks_gen()]
sheet.column += ['Patch creation'] + rownames + ['Patch application'
] + rownames
sheet.column += [''] * (len(rownames) + 1) + ['AVG'] + [''] * len(rownames)
sheet.column += [''] * (len(rownames) + 1) + ['MAX'] + [''] * len(rownames)
for seeds in support.all_seeds_gen():
# Empty cell
times = ['']
for (benchmark, name) in support.benchmarks_gen():
# Determine all paths
base_data = os.path.join(
support.create_path_for_seeds(config.data_dir), benchmark)
div_symfile_path = os.path.join(
support.create_path_for_seeds(config.data_dir, *seeds),
benchmark, 'symfile')
# Time patch creation
def time_function1():
patch.patch(base_data,
seeds,
div_symfile_path=div_symfile_path,
output_dir=config.tmp_dir)
times.append(timeit.timeit(time_function1, number=1))
shutil.copyfile(os.path.join(config.tmp_dir, 'patch'),
os.path.join(config.tmp_dir, 'patch.' + name))
# Empty cell
times.append('')
for (benchmark, name) in support.benchmarks_gen():
base_data = os.path.join(
support.create_path_for_seeds(config.data_dir), benchmark)
# Time patch application
def time_function2():
patch.patch(base_data,
seeds,
patch_path=os.path.join(config.tmp_dir,
'patch.' + name),
output_dir=config.tmp_dir)
times.append(timeit.timeit(time_function2, number=1))
sheet.column += times
# Calculate in the average and the max
for row in (row for row in sheet.rows() if not row[0].startswith('Patch')):
times = [elem for elem in row[3:] if isinstance(elem, float)]
if times:
row[1] = float(sum(times) / len(times))
row[2] = max(times)
# Create the report book and write it out
report = pyexcel.Book(sheets={'Timing': sheet})
report.save_as(os.path.join(config.reports_dir, 'patch_timing.ods'))
def main():
# We prepare by copying the binaries to the location where they'll be protected
print(
'************ Preparing for link-time protections by copying binaries **********'
)
for link_protections in support.link_subsets_gen(seed.get_types(), False):
for build_protections in support.build_subsets_gen(seed.get_types()):
for build_seeds, link_seeds in zip(
support.seeds_gen(*build_protections),
support.seeds_gen(*link_protections)):
support.copy_spec_tree(
support.create_path_for_seeds(config.build_dir,
*build_seeds),
support.create_path_for_seeds(config.build_dir,
*build_seeds, *link_seeds))
for (benchmark, name) in support.benchmarks_gen():
# Get all the sections from all the objects in the build directory
print('************************* ' + benchmark +
' **********************')
print(
'************************* Gathering sections **********************'
)
linkermap = Map(
os.path.join(support.create_path_for_seeds(config.build_dir),
benchmark, name + '.map'))
# Get all the pre_sections and make linker rules out of them. We do this so that they can't change order (which apparently, they can...).
# Use a '*' (even though it is unnecessary) so Diablo's map parser will recognize it as a pattern.
# Remove the name of the encompassing archive from an object (if it exists), else Diablo can't handle this
pre_sections = [[
support.get_objname(section.obj) + '*(' + section.name + ')'
] for section in linkermap.pre_sections]
# We want to create a list of all linker rules that can be altered. Ideally we would simply take all sections currently in the
# linkermap (that we want to change) and convert them into a rule that matches only that section from its specific object.
# Unfortunately the linker is a bit fickle in its handling of weak symbols. If N sections (coming from N different objects)
# exist that define the same symbol, the linker will select only one (from one object) to place in the binary and discard
# the rest. The problem is that during the second, protected link (using the linker script we generate here) the
# linker won't necessarily select a weak symbol section from the same object as it did in the first link. The custom rule
# (which includes the name of the object the section came from the first link) won't match and, for example, the section would
# be put after the sections that ARE shuffled. To avoid this, we also keep all discarded sections, and then create N rules for the
# section (one for each object). These rules stay together during the protections, thus guaranteeing the right location of the section.
main_sections = []
for section in linkermap.main_sections:
rule_list = []
# Create the linker rules and insert them in the list
# Use a '*' (even though it is unnecessary) so Diablo's map parser will recognize it as a pattern.
# Remove the name of the encompassing archive from an object (if it exists), else Diablo can't handle this
suffix = '*(' + section.name + ')'
rule_list.append(support.get_objname(section.obj) + suffix)
for discarded in linkermap.discarded_sections:
if discarded.name == section.name:
rule_list.append(
support.get_objname(discarded.obj) + suffix)
# Add the rule list to the list of lists
main_sections.append(rule_list)
# Perform the actual link-time protections by creating a new linker script (in which sections can change order) and relinking
for link_protections in support.link_subsets_gen(
seed.get_types(), False):
# First create a new linker script for every combination of link protections
for link_seeds in support.seeds_gen(*link_protections):
print('************ Protecting binary at link level for ' +
' '.join([repr(s)
for s in link_seeds]) + ' ... **********')
# Copy the list so that every time we start from the original array
protected_pre_sections = list(pre_sections)
protected_main_sections = list(main_sections)
for s in link_seeds:
protected_pre_sections, protected_main_sections = s.diversify_link(
protected_pre_sections, protected_main_sections)
# Create diversified link script
linker.create_linker_script(
protected_pre_sections + protected_main_sections,
os.path.join(
support.create_path_for_seeds(config.build_dir,
*link_seeds), benchmark,
'link.xc'))
for build_protections in support.build_subsets_gen(
seed.get_types()):
for build_seeds, link_seeds in zip(
support.seeds_gen(*build_protections),
support.seeds_gen(*link_protections)):
# Get the link command, then adapt it to use our new linker script
directory = os.path.join(
support.create_path_for_seeds(config.build_dir,
*build_seeds,
*link_seeds), benchmark)
with open(os.path.join(directory, 'make.out'), 'r') as f:
cmd = list(f)[-1].rstrip()
new_script = os.path.join(
support.create_path_for_seeds(config.build_dir,
*link_seeds), benchmark,
'link.xc')
cmd = cmd.replace(config.link_script, new_script)
# Execute them with our diversified linker script
subprocess.check_call(shlex.split(cmd), cwd=directory)
