def handle_multiple(self, args):
"""
Sequentially run multiple simulations. Note the multiprocess command
is much faster. Only use this command if you have time!
"""
with Timer() as timer:
sims = [ConsistencySimulation.load(fobj, trace=args.trace, outages=args.outages) for fobj in args.data]
output = [StringIO() for sim in sims]
for idx, sim in enumerate(sims):
sim.logger.info("Starting simulation loaded from {!r}".format(args.data[idx].name))
sim.run()
sim.results.dump(output[idx])
output = [json.loads(o.getvalue()) for o in output]
duration = sum([result["timer"]["finished"] - result["timer"]["started"] for result in output])
# Dump the output data to a file.
if args.output is None:
args.output = self.get_output_path(sims[0].name + "-multi")
json.dump(output, args.output)
return ("{} simulations ({} seconds) run in {}\n" "Results for {} written to {}").format(
len(sims), duration, timer, output[0]["simulation"], args.output.name
)
def create_trace(topof, num_nodes, num_locs=N_LOCATIONS, num_objs=N_OBJECTS,
conflict=CONFLICT, Am=ACCESS_MU, As=ACCESS_SIGMA,
ts=TIMESTEPS):
# Modify parameters
# Disable logging during trace generation
logger = logging.getLogger('cloudscope.simulation')
logger.disabled = True
# Update settings arguments
settings.simulation.conflict_prob = conflict
settings.simulation.access_mean = Am
settings.simulation.access_stddev = As
# Simulation arguments
kwargs = {
'users': num_nodes,
'objects': num_objs,
'max_sim_time': ts,
'trace': None,
}
# Create simulation
simulation = ConsistencySimulation.load(topof, **kwargs)
simulation.trace = None
simulation.script()
workload = simulation.workload
# Create the traces writer and write the traces to disk
writer = TracesWriter(workload, ts)
outpath = os.path.join(TRACES, TRACENAME.format(num_nodes, num_locs))
with open(outpath, 'w') as f:
counts = writer.write(f)
def handle_single(self, args):
"""
Most common use case for the simulation runner: simply runs a single
simulation, loading it from the data file.
"""
sim = ConsistencySimulation.load(args.data[0],
trace=args.trace,
outages=args.outages)
sim.run()
# Dump the output data to a file.
if args.output is None:
args.output = self.get_output_path(sim.name, sim.results.finished)
sim.results.dump(args.output)
# Print out the consistency report.
if args.consistency_report:
print "\nPer Log Consistency Report"
print sim.results.consistency.log_inconsistency_table()
print "\nSplit Log Distance Report (Jaccard and Levenshtein)"
print sim.results.consistency.split_log_distance_table()
print
return "Results for {} written to {}".format(sim.name,
args.output.name)
def handle(self, args):
"""
Uses built in workloads and the TracesWriter to generate a trace file.
"""
# Disable logging during trace generation
logger = logging.getLogger('cloudscope.simulation')
logger.disabled = True
# Update settings arguments
settings.simulation.conflict_prob = args.conflict
settings.simulation.access_mean = args.access_mean
settings.simulation.access_stddev = args.access_stddev
# Simulation arguments
kwargs = {
'users': args.users,
'objects': args.objects,
'max_sim_time': args.timesteps,
'trace': None,
}
# Create simulation
simulation = ConsistencySimulation.load(args.data[0], **kwargs)
simulation.trace = None
# Create or select the correct simulation
if args.best_case or args.tiered:
workload = BestCaseAllocation(
simulation,
args.objects,
selection='random',
)
workload.allocate_many(args.users)
elif args.ping_pong:
factory = CharacterSequence(upper=True)
objects = [factory.next() for _ in range(args.objects)]
workload = PingPongWorkload(
simulation,
simulation.replicas[:args.users],
objects=objects,
)
else:
simulation.script()
workload = simulation.workload
# Create the traces writer and write the traces to disk
writer = TracesWriter(workload, args.timesteps)
counts = writer.write(args.output)
return (
"traced {rows:,} accesses on {devices:,} devices over {timesteps:,} timesteps ({realtime})\n"
"object space contains {objects:,} object names:\n"
" {mean_objects_per_device:,} average objects per device | "
"{mean_devices_per_object:,} average devices per object\n"
" {mean_accesses_per_device:,} average accesses per device | "
"{mean_accesses_per_object:,} average accesses per object\n"
"wrote the trace file to {0}").format(args.output.name, **counts)
def load_simulation(path, **kwargs):
"""
Helper function for loading a simulation consistently.
"""
defaults = {"max_sim_time": 100000, "objects": 10, "users": 3}
defaults.update(kwargs)
with open(path, "r") as fobj:
return ConsistencySimulation.load(fobj, **defaults)
def handle(self, args):
"""
Uses built in workloads and the TracesWriter to generate a trace file.
"""
# Disable logging during trace generation
logger = logging.getLogger('cloudscope.simulation')
logger.disabled = True
# Update settings arguments
settings.simulation.conflict_prob = args.conflict
settings.simulation.access_mean = args.access_mean
settings.simulation.access_stddev = args.access_stddev
# Simulation arguments
kwargs = {
'users': args.users,
'objects': args.objects,
'max_sim_time': args.timesteps,
'trace': None,
}
# Create simulation
simulation = ConsistencySimulation.load(args.data[0], **kwargs)
simulation.trace = None
# Create or select the correct simulation
if args.best_case or args.tiered:
workload = BestCaseAllocation(
simulation, args.objects, selection='random',
)
workload.allocate_many(args.users)
elif args.ping_pong:
factory = CharacterSequence(upper=True)
objects = [factory.next() for _ in range(args.objects)]
workload = PingPongWorkload(
simulation, simulation.replicas[:args.users], objects=objects,
)
else:
simulation.script()
workload = simulation.workload
# Create the traces writer and write the traces to disk
writer = TracesWriter(workload, args.timesteps)
counts = writer.write(args.output)
return (
"traced {rows:,} accesses on {devices:,} devices over {timesteps:,} timesteps ({realtime})\n"
"object space contains {objects:,} object names:\n"
" {mean_objects_per_device:,} average objects per device | "
"{mean_devices_per_object:,} average devices per object\n"
" {mean_accesses_per_device:,} average accesses per device | "
"{mean_accesses_per_object:,} average accesses per object\n"
"wrote the trace file to {0}"
).format(args.output.name, **counts)
def load_simulation(path, **kwargs):
"""
Helper function for loading a simulation consistently.
"""
defaults = {
'max_sim_time': 100000,
'objects': 10,
'users': 3,
}
defaults.update(kwargs)
with open(path, 'r') as fobj:
return ConsistencySimulation.load(fobj, **defaults)
def getsim(topology='tag', **kwargs):
# Find the correct topology
topology = {
'tag': TAG,
'raft': RAFT,
'eventual': EVENTUAL,
}[topology.lower()]
# Update the default options
options = OPTIONS.copy()
options.update(kwargs)
# Load the simulation
with open(topology, 'r') as fobj:
sim = ConsistencySimulation.load(fobj, **options)
return sim
def getsim(topology='tag', **kwargs):
# Find the correct topology
topology = {
'tag': TAG,
'raft': RAFT,
'eventual': EVENTUAL,
}[topology.lower()]
# Update the default options
options = OPTIONS.copy()
options.update(kwargs)
# Load the simulation
with open(topology, 'r') as fobj:
sim = ConsistencySimulation.load(fobj, **options)
return sim
def runner(idx, path, **kwargs):
"""
Takes an open file-like object and runs the simulation, returning a
string containing the dumped JSON results, which can be dumped to disk.
"""
logger = logging.getLogger('cloudscope')
try:
with open(path, 'r') as fobj:
sim = ConsistencySimulation.load(fobj, **kwargs)
logger.info("Starting simulation {}: \"{}\"".format(idx, sim.name))
sim.run()
# Dump the output data to a file.
output = StringIO()
sim.results.dump(output)
# Report completion back to the command line
logger.info("Simulation {}: \"{}\" completed in {}".format(
idx, sim.name, sim.results.timer))
# Return the string value of the JSON results.
return output.getvalue()
except Exception as e:
logger.error("Simulation {} ({}) errored: {}".format(
idx, path, str(e)))
import traceback, sys
return json.dumps({
'idx':
idx,
'success':
False,
'traceback':
"".join(traceback.format_exception(*sys.exc_info())),
'error':
str(e),
})
def runner(idx, path, **kwargs):
"""
Takes an open file-like object and runs the simulation, returning a
string containing the dumped JSON results, which can be dumped to disk.
"""
logger = logging.getLogger('cloudscope')
try:
with open(path, 'r') as fobj:
sim = ConsistencySimulation.load(fobj, **kwargs)
logger.info(
"Starting simulation {}: \"{}\"".format(idx, sim.name)
)
sim.run()
# Dump the output data to a file.
output = StringIO()
sim.results.dump(output)
# Report completion back to the command line
logger.info(
"Simulation {}: \"{}\" completed in {}".format(idx, sim.name, sim.results.timer)
)
# Return the string value of the JSON results.
return output.getvalue()
except Exception as e:
logger.error(
"Simulation {} ({}) errored: {}".format(idx, path, str(e))
)
import traceback, sys
return json.dumps({
'idx': idx,
'success': False,
'traceback': "".join(traceback.format_exception(*sys.exc_info())),
'error': str(e),
})
def handle_single(self, args):
"""
Most common use case for the simulation runner: simply runs a single
simulation, loading it from the data file.
"""
sim = ConsistencySimulation.load(args.data[0], trace=args.trace, outages=args.outages)
sim.run()
# Dump the output data to a file.
if args.output is None:
args.output = self.get_output_path(sim.name, sim.results.finished)
sim.results.dump(args.output)
# Print out the consistency report.
if args.consistency_report:
print "\nPer Log Consistency Report"
print sim.results.consistency.log_inconsistency_table()
print "\nSplit Log Distance Report (Jaccard and Levenshtein)"
print sim.results.consistency.split_log_distance_table()
print
return "Results for {} written to {}".format(sim.name, args.output.name)
def handle_multiple(self, args):
"""
Sequentially run multiple simulations. Note the multiprocess command
is much faster. Only use this command if you have time!
"""
with Timer() as timer:
sims = [
ConsistencySimulation.load(fobj,
trace=args.trace,
outages=args.outages)
for fobj in args.data
]
output = [StringIO() for sim in sims]
for idx, sim in enumerate(sims):
sim.logger.info("Starting simulation loaded from {!r}".format(
args.data[idx].name))
sim.run()
sim.results.dump(output[idx])
output = [json.loads(o.getvalue()) for o in output]
duration = sum([
result['timer']['finished'] - result['timer']['started']
for result in output
])
# Dump the output data to a file.
if args.output is None:
args.output = self.get_output_path(sims[0].name + "-multi")
json.dump(output, args.output)
return ("{} simulations ({} seconds) run in {}\n"
"Results for {} written to {}").format(len(sims), duration,
timer,
output[0]['simulation'],
args.output.name)
def handle(self, args):
"""
Uses the OutagesWriter to generate an outages script.
"""
# Disable logging during trace generation
logger = logging.getLogger('cloudscope.simulation')
logger.disabled = True
# Simulation arguments
kwargs = {
'max_sim_time': args.timesteps,
'outages': None,
}
# Create simulation
simulation = ConsistencySimulation.load(args.data[0], **kwargs)
simulation.script()
# Create the traces writer and write the traces to disk
writer = OutagesWriter(simulation.partitions, args.timesteps)
rows = writer.write(args.output)
return (
"scripted {} outages on {} connections by {} generators "
"over {} timesteps\n"
"Outage probability was {} partitioned by '{}' strategy\n"
"Online Distribution (ms) = {}μ {}σ\n"
"Outage Distribution (ms) = {}μ {}σ\n"
"wrote the script file to {}"
).format(
rows, sum(len(gen.connections) for gen in simulation.partitions),
len(simulation.partitions), args.timesteps,
settings.simulation.outage_prob, settings.simulation.partition_across,
settings.simulation.online_mean, settings.simulation.online_stddev,
settings.simulation.outage_mean, settings.simulation.outage_stddev,
args.output.name
)
def handle(self, args):
"""
Uses the OutagesWriter to generate an outages script.
"""
# Disable logging during trace generation
logger = logging.getLogger('cloudscope.simulation')
logger.disabled = True
# Simulation arguments
kwargs = {
'max_sim_time': args.timesteps,
'outages': None,
}
# Create simulation
simulation = ConsistencySimulation.load(args.data[0], **kwargs)
simulation.script()
# Create the traces writer and write the traces to disk
writer = OutagesWriter(simulation.partitions, args.timesteps)
rows = writer.write(args.output)
return ("scripted {} outages on {} connections by {} generators "
"over {} timesteps\n"
"Outage probability was {} partitioned by '{}' strategy\n"
"Online Distribution (ms) = {}μ {}σ\n"
"Outage Distribution (ms) = {}μ {}σ\n"
"wrote the script file to {}").format(
rows,
sum(len(gen.connections) for gen in simulation.partitions),
len(simulation.partitions), args.timesteps,
settings.simulation.outage_prob,
settings.simulation.partition_across,
settings.simulation.online_mean,
settings.simulation.online_stddev,
settings.simulation.outage_mean,
settings.simulation.outage_stddev, args.output.name)