def table_test(self,
table_prefix="table_test1",
default_vis="default",
num_entries=10000,
num_rows=1000):
table_prefix = sanitize(table_prefix)
generate_data(self.filename,
self.seed,
default_vis=default_vis,
num_entries=num_entries,
num_rows=num_rows)
for signer_id, sigclass in self.signer_ids:
_, privkey = sigclass.test_keys()
table = table_prefix + '_' + sanitize(signer_id)
signer = AccumuloSigner(privkey, sig_f=sigclass)
write_and_sign_data(self.filename,
self.conn,
table,
signer,
include_table=True)
verif_key, _ = self.pki.get_verifying_key(signer_id)
verify_data(self.conn, table, verif_key, False, include_table=True)
def location_test(self,
cfg_file,
table_prefix="table_test1",
default_vis="default",
num_entries=10000,
num_rows=1000):
table_prefix = sanitize(table_prefix) + '_' + sanitize(loc)
generate_data(self.filename,
self.seed,
default_vis=default_vis,
num_entries=num_entries,
num_rows=num_rows)
for signer_id, sigclass in self.signer_ids:
_, privkey = sigclass.test_keys()
table = table_prefix + '_' + sanitize(signer_id)
conf = new_config(cfg_file, self.conn)
signer = AccumuloSigner(privkey, sig_f=sigclass, conf=conf)
write_and_sign_data(self.filename, self.conn, table, signer)
verif_key, _ = self.pki.get_verifying_key(signer_id)
verify_data(self.conn, table, verif_key, False, conf=conf)
def run_benchmarks(self,
table_prefix="benchmarking",
default_vis="default"):
""" Benchmarks each different signature class on a variety of table
sizes, measuring the time taken to sign & verify all entries of each
table size with each signature algorithm.
"""
table_prefix = sanitize(table_prefix)
for entries, rows in self.BENCHMARKS:
print "==============================================================="
print "Current benchmark: %d entries over %d rows" % (entries,
rows)
print "==============================================================="
print
for signClass in SUPPORTED_SIGNATURES:
table = "%s_%s_e%d_r%d" % (
table_prefix, sanitize(signClass.name), entries, rows)
print "Benchmarking %s" % (sanitize(signClass.name))
self.run_test(table,
default_vis,
entries,
rows,
signClass.name,
write=True,
benchmark=True)
print
def run_test(self,
table="test_table_5",
default_vis="default",
num_entries=100,
num_rows=15,
signClassName='RSASSA_PKCS1-v1_5',
write=True,
benchmark=False):
""" Runs one iteration of the signature test. If benchmark is set to
True, returns the lengths of time it took to sign all the entries
and the time it took to verify all the entries.
"""
table = sanitize(table)
seed = self.seed
if signClassName == 'ALL':
for signClass in ALL_SIGNATURES:
self.run_test(table + '_' + sanitize(signClass.name),
default_vis, num_entries, num_rows,
signClass.name, write, benchmark)
return
signClass = SIGNATURE_FUNCTIONS[signClassName]
pubkey, privkey = signClass.test_keys()
if write:
signer = AccumuloSigner(privkey, sig_f=signClass)
if not seed:
# set a new seed if one wasn't specified
seed = str(time.time())
generate_data(self.filename,
seed,
default_vis=default_vis,
num_entries=num_entries,
num_rows=num_rows)
sout = write_and_sign_data(self.filename, self.conn, table, signer,
benchmark)
vout = verify_data(self.conn, table, pubkey, benchmark)
if benchmark:
sign_start, sign_end = sout
verif_success, verif_start, verif_end = vout
print "Time taken to sign: %s" % str(sign_end - sign_start)
print "Time taken to verify: %s" % str(verif_end - verif_start)
return sign_end - sign_start, verif_end - verif_start
def full_benchmark(self,
table_prefix="full_benchmarking",
default_vis="default",
signClass=None,
num_entries=10000,
num_rows=1000):
""" Either run a single benchmark (sign & verify) on one signature
class, or run it with no signing class (just write & read) to get
a baseline time.
"""
table_prefix = sanitize(table_prefix)
conn = self.conn
if signClass:
table = table_prefix + '_' + sanitize(signClass.name)
else:
table = table_prefix + '_baseline'
if signClass:
pubkey, privkey = signClass.test_keys()
signer = AccumuloSigner(privkey, sig_f=signClass)
start_time = time.clock()
write_and_sign_data(self.filename,
conn,
table,
signer,
benchmark=False)
end_time = time.clock()
total_sign_time = end_time - start_time
start_time = time.clock()
verify_data(conn, table, pubkey, benchmark=False)
end_time = time.clock()
total_verif_time = end_time - start_time
else:
start_time = time.clock()
write_data(self.filename, conn, table)
end_time = time.clock()
total_sign_time = end_time - start_time
count = 0
start_time = time.clock()
for entry in conn.scan(table):
count += 1
end_time = time.clock()
total_verif_time = end_time - start_time
return (total_sign_time, total_verif_time)
def run_fancy_benchmarks(self,
table_prefix="benchmarking",
default_vis="default",
resfile="benchmark_results.csv"):
""" Runs more benchmarks than run_benchmarks(), then writes the output
to a file.
"""
table_prefix = sanitize(table_prefix)
results = []
for entries, rows in self.FANCY_BENCHMARKS:
print "==============================================================="
print "Current benchmark: %d entries over %d rows" % (entries,
rows)
print "==============================================================="
print
classres = []
for signClass in SUPPORTED_SIGNATURES:
table = "%s_%s_e%d_r%d" % (
table_prefix, sanitize(signClass.name), entries, rows)
print "Benchmarking %s" % (sanitize(signClass.name))
sign_time, verif_time = self.run_test(table,
default_vis,
entries,
rows,
signClass.name,
write=True,
benchmark=True)
classres.append((signClass.name, sign_time, verif_time))
print
results.append((entries, classres))
print 'time to write to file'
with open(resfile, 'w') as f:
f.write('num entries,name,sign time,verification time\n')
for num_entries, classres in results:
for name, stime, vtime in classres:
f.write(','.join(
[str(num_entries), name,
str(stime),
str(vtime)]))
f.write('\n')
print 'wrote to file'
def id_test(self,
table_prefix="id_test",
default_vis="default",
num_entries=10000,
num_rows=1000):
table_prefix = sanitize(table_prefix)
generate_data(self.filename,
self.seed,
default_vis=default_vis,
num_entries=num_entries,
num_rows=num_rows)
for signer_id, sigclass in self.signer_ids:
_, privkey = sigclass.test_keys()
table = table_prefix + '_' + sanitize(signer_id)
signer = AccumuloSigner(privkey,
sig_f=sigclass,
signerID=signer_id)
write_and_sign_data(self.filename, self.conn, table, signer)
verify_data(self.conn, table, self.pki, sigclass)
def fastfail_benchmark(self, table):
""" Check how long it takes just to read each element from a table,
to see if there's a difference because of the changed visibility
fields in signed tables.
"""
table = sanitize(table)
start = time.clock()
total = 0
for e in self.conn.scan(table):
total += 1
end = time.clock()
return end - start
def run_full_benchmarks(self,
table_prefix="full_benchmarking",
default_vis="default",
num_entries=10000,
num_rows=1000,
outfile='full_benchmark_out.csv'):
""" Benchmark each signing algorithm, writing the results to a file,
and comparing them to a baseline write & read with no signatures.
"""
table_prefix = sanitize(table_prefix)
n = generate_data(self.filename,
self.seed,
default_vis=default_vis,
num_entries=num_entries,
num_rows=num_rows)
base_write_time, base_read_time = self.full_benchmark(
table_prefix, default_vis, None, num_entries, num_rows)
with open(outfile, 'w') as f:
bw = (base_write_time / n) * 1000
br = (base_read_time / n) * 1000
f.write(','.join(['name', 'signing time', 'verification time']))
f.write('\n')
f.write(','.join(['baseline', str(bw), str(br)]))
f.write('\n')
for signClass in SUPPORTED_SIGNATURES:
(st, vt) = self.full_benchmark(table_prefix, default_vis,
signClass, num_entries,
num_rows)
# convert seconds for the whole batch to milliseconds
# per element
st = (st / n) * 1000
vt = (vt / n) * 1000
f.write(','.join([signClass.name, str(st), str(vt)]))
f.write('\n')
def run_fastfail_benchmarks(self,
table_prefix="fastfail_benchmarking",
default_vis="default",
num_rows=1000,
num_noisy_entries=50000,
num_noisy_rows=1000,
outfile='fastfail_benchmark_out_2.csv',
num_trials=100,
one_vis=False):
""" Benchmark to see how much overhead there is from the signature code
making Accumulo unable to fast-fail and cache results from
visibility field checks.
If one_vis is False, it will randomly generate a default visibility
value for each field. If it is a string, that string will be treated
as the default visibility value for each 'noise' field.
"""
table_prefix = sanitize(table_prefix)
seed = self.seed
noisy_filename = 'noisy_' + self.filename
if not seed:
# set a new seed if one wasn't specified
seed = str(time.time())
if one_vis:
print 'generating noise with one visibility field'
generate_data(noisy_filename,
seed,
vis=False,
default_vis=one_vis,
num_entries=num_noisy_entries,
num_rows=num_rows)
else:
print 'generating noise with random visibility fields'
generate_data(noisy_filename,
seed,
vis=True,
num_entries=num_noisy_entries,
num_rows=num_rows)
noisy_table = 'noisy_' + table_prefix
write_data(noisy_filename, self.conn, noisy_table)
for sc in SUPPORTED_SIGNATURES:
pubkey, privkey = sc.test_keys()
signer = AccumuloSigner(privkey, sig_f=sc)
write_and_sign_data(noisy_filename, self.conn,
'_'.join([table_prefix,
sanitize(sc.name)]), signer)
all_times = []
for n in [(num_noisy_entries / 10000) * (10**i) for i in range(6)]:
print 'n:', n
generate_data(self.filename,
str(time.time()),
default_vis=default_vis,
num_entries=n,
num_rows=min(n, num_rows))
write_data(self.filename, self.conn, noisy_table)
base_time = sum([
self.fastfail_benchmark(noisy_table) for j in range(num_trials)
])
times = []
for signClass in SUPPORTED_SIGNATURES:
pubkey, privkey = signClass.test_keys()
signer = AccumuloSigner(privkey, sig_f=signClass)
table = '_'.join([table_prefix, sanitize(signClass.name)])
write_and_sign_data(self.filename, self.conn, table, signer)
times.append((signClass.name,
sum([
self.fastfail_benchmark(table)
for j in range(num_trials)
])))
all_times.append((n, base_time, times))
with open(outfile, 'w') as f:
for num_elems, base_time, trials in all_times:
print 'Trial for %d elements. Base time: %s' % (num_elems,
str(base_time))
f.write('%d,BASE,%s\n' % (num_elems, str(base_time)))
for name, ttime in trials:
print '\t%s: %s' % (name, str(ttime))
f.write('%d,%s,%s\n' % (num_elems, name, str(ttime)))
print