def write_stats(ranges, results, prefix="stats"):
attrs = list(ranges.keys())
df = DataFile(
os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))),
['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] +
attr_names("iter time") + attr_names("iter thruput") +
attr_names("file size") + attr_names("scan thruput") + attrs,
overwrite=True)
for config, reps in results:
cfg = dict(config)
# TODO: avg. over the reps
iter_times = Entry("iter time")
iter_tps = Entry("iter thruput")
file_sizes = Entry("file size")
scan_tps = Entry("scan thruput")
for data in reps:
db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(
", ")
iter_times.add(int(iter_time))
iter_tps.add(int(iter_tp))
file_sizes.add(int(os.path.getsize(db_path)))
scan_tps.add(float(scans_time) / float(num_scans))
for attr in ranges:
df[attr] = cfg[attr]
df['entries'] = entries
df['lookups'] = lookups
df['hits'] = hits
df['total time'] = total_time
df['lookup time'] = lookup_time
df['scans'] = num_scans
# df['file size'] = os.path.getsize(db_path)
for attr in attr_names("iter time"):
df[attr] = getattr(iter_times, attr.split()[-1])
for attr in attr_names("iter thruput"):
df[attr] = getattr(iter_tps, attr.split()[-1])
for attr in attr_names("file size"):
df[attr] = getattr(file_sizes, attr.split()[-1])
for attr in attr_names("scan thruput"):
df[attr] = getattr(scan_tps, attr.split()[-1])
print file_sizes
print iter_times
print iter_tps
print scan_tps
df.save()
df.close()
def write_stats(ranges, results, prefix="stats"):
attrs = list(ranges.keys())
df = DataFile(os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))), ['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] + attr_names("iter time") + attr_names("iter thruput") + attr_names("file size") + attr_names("scan thruput") + attrs, overwrite=True)
for config, reps in results:
cfg = dict(config)
# TODO: avg. over the reps
iter_times = Entry("iter time")
iter_tps = Entry("iter thruput")
file_sizes = Entry("file size")
scan_tps = Entry("scan thruput")
for data in reps:
db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(", ")
iter_times.add(int(iter_time))
iter_tps.add(int(iter_tp))
file_sizes.add(int(os.path.getsize(db_path)))
scan_tps.add(float(scans_time)/float(num_scans))
for attr in ranges:
df[attr] = cfg[attr]
df['entries'] = entries
df['lookups'] = lookups
df['hits'] = hits
df['total time'] = total_time
df['lookup time'] = lookup_time
df['scans'] = num_scans
# df['file size'] = os.path.getsize(db_path)
for attr in attr_names("iter time"):
df[attr] = getattr(iter_times, attr.split()[-1])
for attr in attr_names("iter thruput"):
df[attr] = getattr(iter_tps, attr.split()[-1])
for attr in attr_names("file size"):
df[attr] = getattr(file_sizes, attr.split()[-1])
for attr in attr_names("scan thruput"):
df[attr] = getattr(scan_tps, attr.split()[-1])
print file_sizes
print iter_times
print iter_tps
print scan_tps
df.save()
df.close()
def exec_ranges(cmd,
args,
range_attrs,
num_reps=10,
pbs_queue=None,
pbs_name='storagesim',
config=None):
# range_attrs is in the form:
# range_val: [list of possible values]
#
# when several range_vals and value lists are given, all possible
# permutations are executed
base_path = args['base_dir']
if config != None:
df = DataFile(os.path.join(config, 'config_ranges.dat'),
list(range_attrs.keys()),
overwrite=True)
outputs = []
for attr_p in permutations(range_attrs):
# set the attrs accordingly for this exec
names = []
for attr, val in attr_p:
args[attr] = val
names.append('%s_%s' % (str(attr), str(val)))
if config != None:
df[attr] = val
if config != None:
df.save()
args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names)))
if not os.path.exists(args['base_dir']):
os.mkdir(args['base_dir'])
outputs.append((attr_p,
exec_repetitions(cmd,
args,
num_reps=num_reps,
pbs_queue=pbs_queue,
pbs_name=pbs_name)))
if config != None:
df.close()
return outputs
def exec_ranges(cmd, args, range_attrs, num_reps=10, pbs_queue=None, pbs_name='storagesim', config=None):
# range_attrs is in the form:
# range_val: [list of possible values]
#
# when several range_vals and value lists are given, all possible
# permutations are executed
base_path = args['base_dir']
if config != None:
df = DataFile(os.path.join(config, 'config_ranges.dat'), list(range_attrs.keys()), overwrite=True)
outputs = []
for attr_p in permutations(range_attrs):
# set the attrs accordingly for this exec
names = []
for attr, val in attr_p:
args[attr] = val
names.append('%s_%s' % (str(attr), str(val)))
if config != None:
df[attr] = val
if config != None:
df.save()
args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names)))
if not os.path.exists(args['base_dir']):
os.mkdir(args['base_dir'])
outputs.append((attr_p, exec_repetitions(cmd, args, num_reps=num_reps, pbs_queue=pbs_queue, pbs_name=pbs_name)))
if config != None:
df.close()
return outputs
