Пример #1
0
    def handle_single(self, args):
        """
        Most common use case for the simulation runner: simply runs a single
        simulation, loading it from the data file.
        """
        sim = ConsistencySimulation.load(args.data[0],
                                         trace=args.trace,
                                         outages=args.outages)
        sim.run()

        # Dump the output data to a file.
        if args.output is None:
            args.output = self.get_output_path(sim.name, sim.results.finished)
        sim.results.dump(args.output)

        # Print out the consistency report.
        if args.consistency_report:
            print "\nPer Log Consistency Report"
            print sim.results.consistency.log_inconsistency_table()
            print "\nSplit Log Distance Report (Jaccard and Levenshtein)"
            print sim.results.consistency.split_log_distance_table()
            print

        return "Results for {} written to {}".format(sim.name,
                                                     args.output.name)
Пример #2
0
def create_trace(topof, num_nodes, num_locs=N_LOCATIONS, num_objs=N_OBJECTS,
                 conflict=CONFLICT, Am=ACCESS_MU, As=ACCESS_SIGMA,
                 ts=TIMESTEPS):

    # Modify parameters
    # Disable logging during trace generation
    logger = logging.getLogger('cloudscope.simulation')
    logger.disabled = True

    # Update settings arguments
    settings.simulation.conflict_prob = conflict
    settings.simulation.access_mean   = Am
    settings.simulation.access_stddev = As

    # Simulation arguments
    kwargs = {
        'users': num_nodes,
        'objects': num_objs,
        'max_sim_time': ts,
        'trace': None,
    }

    # Create simulation
    simulation = ConsistencySimulation.load(topof, **kwargs)
    simulation.trace = None
    simulation.script()
    workload = simulation.workload

    # Create the traces writer and write the traces to disk
    writer = TracesWriter(workload, ts)
    outpath = os.path.join(TRACES, TRACENAME.format(num_nodes, num_locs))

    with open(outpath, 'w') as f:
        counts = writer.write(f)
Пример #3
0
    def handle_multiple(self, args):
        """
        Sequentially run multiple simulations. Note the multiprocess command
        is much faster. Only use this command if you have time!
        """
        with Timer() as timer:

            sims = [ConsistencySimulation.load(fobj, trace=args.trace, outages=args.outages) for fobj in args.data]
            output = [StringIO() for sim in sims]

            for idx, sim in enumerate(sims):
                sim.logger.info("Starting simulation loaded from {!r}".format(args.data[idx].name))
                sim.run()
                sim.results.dump(output[idx])

            output = [json.loads(o.getvalue()) for o in output]

            duration = sum([result["timer"]["finished"] - result["timer"]["started"] for result in output])

            # Dump the output data to a file.
            if args.output is None:
                args.output = self.get_output_path(sims[0].name + "-multi")
            json.dump(output, args.output)

        return ("{} simulations ({} seconds) run in {}\n" "Results for {} written to {}").format(
            len(sims), duration, timer, output[0]["simulation"], args.output.name
        )
Пример #4
0
    def handle(self, args):
        """
        Uses built in workloads and the TracesWriter to generate a trace file.
        """
        # Disable logging during trace generation
        logger = logging.getLogger('cloudscope.simulation')
        logger.disabled = True

        # Update settings arguments
        settings.simulation.conflict_prob = args.conflict
        settings.simulation.access_mean = args.access_mean
        settings.simulation.access_stddev = args.access_stddev

        # Simulation arguments
        kwargs = {
            'users': args.users,
            'objects': args.objects,
            'max_sim_time': args.timesteps,
            'trace': None,
        }

        # Create simulation
        simulation = ConsistencySimulation.load(args.data[0], **kwargs)
        simulation.trace = None

        # Create or select the correct simulation
        if args.best_case or args.tiered:
            workload = BestCaseAllocation(
                simulation,
                args.objects,
                selection='random',
            )
            workload.allocate_many(args.users)

        elif args.ping_pong:
            factory = CharacterSequence(upper=True)
            objects = [factory.next() for _ in range(args.objects)]
            workload = PingPongWorkload(
                simulation,
                simulation.replicas[:args.users],
                objects=objects,
            )

        else:
            simulation.script()
            workload = simulation.workload

        # Create the traces writer and write the traces to disk
        writer = TracesWriter(workload, args.timesteps)
        counts = writer.write(args.output)

        return (
            "traced {rows:,} accesses on {devices:,} devices over {timesteps:,} timesteps ({realtime})\n"
            "object space contains {objects:,} object names:\n"
            "  {mean_objects_per_device:,} average objects per device | "
            "{mean_devices_per_object:,} average devices per object\n"
            "  {mean_accesses_per_device:,} average accesses per device | "
            "{mean_accesses_per_object:,} average accesses per object\n"
            "wrote the trace file to {0}").format(args.output.name, **counts)
Пример #5
0
def load_simulation(path, **kwargs):
    """
    Helper function for loading a simulation consistently.
    """
    defaults = {"max_sim_time": 100000, "objects": 10, "users": 3}
    defaults.update(kwargs)

    with open(path, "r") as fobj:
        return ConsistencySimulation.load(fobj, **defaults)
Пример #6
0
    def handle(self, args):
        """
        Uses built in workloads and the TracesWriter to generate a trace file.
        """
        # Disable logging during trace generation
        logger = logging.getLogger('cloudscope.simulation')
        logger.disabled = True

        # Update settings arguments
        settings.simulation.conflict_prob = args.conflict
        settings.simulation.access_mean   = args.access_mean
        settings.simulation.access_stddev = args.access_stddev

        # Simulation arguments
        kwargs = {
            'users': args.users,
            'objects': args.objects,
            'max_sim_time': args.timesteps,
            'trace': None,
        }

        # Create simulation
        simulation = ConsistencySimulation.load(args.data[0], **kwargs)
        simulation.trace = None

        # Create or select the correct simulation
        if args.best_case or args.tiered:
            workload = BestCaseAllocation(
                simulation, args.objects, selection='random',
            )
            workload.allocate_many(args.users)

        elif args.ping_pong:
            factory = CharacterSequence(upper=True)
            objects = [factory.next() for _ in range(args.objects)]
            workload = PingPongWorkload(
                simulation, simulation.replicas[:args.users], objects=objects,
            )

        else:
            simulation.script()
            workload = simulation.workload

        # Create the traces writer and write the traces to disk
        writer = TracesWriter(workload, args.timesteps)
        counts = writer.write(args.output)

        return (
            "traced {rows:,} accesses on {devices:,} devices over {timesteps:,} timesteps ({realtime})\n"
            "object space contains {objects:,} object names:\n"
            "  {mean_objects_per_device:,} average objects per device | "
            "{mean_devices_per_object:,} average devices per object\n"
            "  {mean_accesses_per_device:,} average accesses per device | "
            "{mean_accesses_per_object:,} average accesses per object\n"
            "wrote the trace file to {0}"
        ).format(args.output.name, **counts)
Пример #7
0
def load_simulation(path, **kwargs):
    """
    Helper function for loading a simulation consistently.
    """
    defaults = {
        'max_sim_time': 100000,
        'objects': 10,
        'users': 3,
    }
    defaults.update(kwargs)

    with open(path, 'r') as fobj:
        return ConsistencySimulation.load(fobj, **defaults)
Пример #8
0
    def setUp(self):
        self.sim = ConsistencySimulation()
        Replica.counter.reset()

        self.alpha = EventualReplica(self.sim)
        self.bravo = EventualReplica(self.sim)
        self.delta = EventualReplica(self.sim)

        self.sim.replicas = [
            self.alpha, self.bravo, self.delta
        ]

        self.versions = []
        self.orig_versioning = settings.simulation.versioning
        settings.simulation.versioning = 'federated'
Пример #9
0
def getsim(topology='tag', **kwargs):
    # Find the correct topology
    topology = {
        'tag': TAG,
        'raft': RAFT,
        'eventual': EVENTUAL,
    }[topology.lower()]

    # Update the default options
    options = OPTIONS.copy()
    options.update(kwargs)

    # Load the simulation
    with open(topology, 'r') as fobj:
        sim = ConsistencySimulation.load(fobj, **options)

    return sim
Пример #10
0
def getsim(topology='tag', **kwargs):
    # Find the correct topology
    topology = {
        'tag': TAG,
        'raft': RAFT,
        'eventual': EVENTUAL,
    }[topology.lower()]

    # Update the default options
    options = OPTIONS.copy()
    options.update(kwargs)

    # Load the simulation
    with open(topology, 'r') as fobj:
        sim = ConsistencySimulation.load(fobj, **options)

    return sim
Пример #11
0
def runner(idx, path, **kwargs):
    """
    Takes an open file-like object and runs the simulation, returning a
    string containing the dumped JSON results, which can be dumped to disk.
    """
    logger = logging.getLogger('cloudscope')

    try:

        with open(path, 'r') as fobj:
            sim = ConsistencySimulation.load(fobj, **kwargs)

        logger.info(
            "Starting simulation {}: \"{}\"".format(idx, sim.name)
        )
        sim.run()

        # Dump the output data to a file.
        output = StringIO()
        sim.results.dump(output)

        # Report completion back to the command line
        logger.info(
            "Simulation {}: \"{}\" completed in {}".format(idx, sim.name, sim.results.timer)
        )

        # Return the string value of the JSON results.
        return output.getvalue()

    except Exception as e:
        logger.error(
            "Simulation {} ({}) errored: {}".format(idx, path, str(e))
        )

        import traceback, sys
        return json.dumps({
            'idx': idx,
            'success': False,
            'traceback': "".join(traceback.format_exception(*sys.exc_info())),
            'error': str(e),
        })
Пример #12
0
def runner(idx, path, **kwargs):
    """
    Takes an open file-like object and runs the simulation, returning a
    string containing the dumped JSON results, which can be dumped to disk.
    """
    logger = logging.getLogger('cloudscope')

    try:

        with open(path, 'r') as fobj:
            sim = ConsistencySimulation.load(fobj, **kwargs)

        logger.info("Starting simulation {}: \"{}\"".format(idx, sim.name))
        sim.run()

        # Dump the output data to a file.
        output = StringIO()
        sim.results.dump(output)

        # Report completion back to the command line
        logger.info("Simulation {}: \"{}\" completed in {}".format(
            idx, sim.name, sim.results.timer))

        # Return the string value of the JSON results.
        return output.getvalue()

    except Exception as e:
        logger.error("Simulation {} ({}) errored: {}".format(
            idx, path, str(e)))

        import traceback, sys
        return json.dumps({
            'idx':
            idx,
            'success':
            False,
            'traceback':
            "".join(traceback.format_exception(*sys.exc_info())),
            'error':
            str(e),
        })
Пример #13
0
    def handle_single(self, args):
        """
        Most common use case for the simulation runner: simply runs a single
        simulation, loading it from the data file.
        """
        sim = ConsistencySimulation.load(args.data[0], trace=args.trace, outages=args.outages)
        sim.run()

        # Dump the output data to a file.
        if args.output is None:
            args.output = self.get_output_path(sim.name, sim.results.finished)
        sim.results.dump(args.output)

        # Print out the consistency report.
        if args.consistency_report:
            print "\nPer Log Consistency Report"
            print sim.results.consistency.log_inconsistency_table()
            print "\nSplit Log Distance Report (Jaccard and Levenshtein)"
            print sim.results.consistency.split_log_distance_table()
            print

        return "Results for {} written to {}".format(sim.name, args.output.name)
Пример #14
0
    def handle_multiple(self, args):
        """
        Sequentially run multiple simulations. Note the multiprocess command
        is much faster. Only use this command if you have time!
        """
        with Timer() as timer:

            sims = [
                ConsistencySimulation.load(fobj,
                                           trace=args.trace,
                                           outages=args.outages)
                for fobj in args.data
            ]
            output = [StringIO() for sim in sims]

            for idx, sim in enumerate(sims):
                sim.logger.info("Starting simulation loaded from {!r}".format(
                    args.data[idx].name))
                sim.run()
                sim.results.dump(output[idx])

            output = [json.loads(o.getvalue()) for o in output]

            duration = sum([
                result['timer']['finished'] - result['timer']['started']
                for result in output
            ])

            # Dump the output data to a file.
            if args.output is None:
                args.output = self.get_output_path(sims[0].name + "-multi")
            json.dump(output, args.output)

        return ("{} simulations ({} seconds) run in {}\n"
                "Results for {} written to {}").format(len(sims), duration,
                                                       timer,
                                                       output[0]['simulation'],
                                                       args.output.name)
Пример #15
0
    def handle(self, args):
        """
        Uses the OutagesWriter to generate an outages script. 
        """
        # Disable logging during trace generation
        logger = logging.getLogger('cloudscope.simulation')
        logger.disabled = True

        # Simulation arguments
        kwargs = {
            'max_sim_time': args.timesteps,
            'outages': None,
        }

        # Create simulation
        simulation = ConsistencySimulation.load(args.data[0], **kwargs)
        simulation.script()

        # Create the traces writer and write the traces to disk
        writer = OutagesWriter(simulation.partitions, args.timesteps)
        rows = writer.write(args.output)

        return ("scripted {} outages on {} connections by {} generators "
                "over {} timesteps\n"
                "Outage probability was {} partitioned by '{}' strategy\n"
                "Online Distribution (ms) = {}μ {}σ\n"
                "Outage Distribution (ms) = {}μ {}σ\n"
                "wrote the script file to {}").format(
                    rows,
                    sum(len(gen.connections) for gen in simulation.partitions),
                    len(simulation.partitions), args.timesteps,
                    settings.simulation.outage_prob,
                    settings.simulation.partition_across,
                    settings.simulation.online_mean,
                    settings.simulation.online_stddev,
                    settings.simulation.outage_mean,
                    settings.simulation.outage_stddev, args.output.name)
Пример #16
0
    def handle(self, args):
        """
        Uses the OutagesWriter to generate an outages script. 
        """
        # Disable logging during trace generation
        logger = logging.getLogger('cloudscope.simulation')
        logger.disabled = True

        # Simulation arguments
        kwargs = {
            'max_sim_time': args.timesteps,
            'outages': None,
        }

        # Create simulation
        simulation = ConsistencySimulation.load(args.data[0], **kwargs)
        simulation.script()

        # Create the traces writer and write the traces to disk
        writer = OutagesWriter(simulation.partitions, args.timesteps)
        rows   = writer.write(args.output)

        return (
            "scripted {} outages on {} connections by {} generators "
            "over {} timesteps\n"
            "Outage probability was {} partitioned by '{}' strategy\n"
            "Online Distribution (ms) = {}μ {}σ\n"
            "Outage Distribution (ms) = {}μ {}σ\n"
            "wrote the script file to {}"
        ).format(
            rows, sum(len(gen.connections) for gen in simulation.partitions),
            len(simulation.partitions), args.timesteps,
            settings.simulation.outage_prob, settings.simulation.partition_across,
            settings.simulation.online_mean, settings.simulation.online_stddev,
            settings.simulation.outage_mean, settings.simulation.outage_stddev,
            args.output.name
        )
Пример #17
0
 def setUp(self):
     self.sim = ConsistencySimulation()
     Replica.counter.reset()