def test_multiprocessing_multiapp_system(self): set_db_uri() with Orders(setup_table=True) as app: # Create a new order. order_id = create_new_order() # Check new order exists in the repository. assert order_id in app.repository system = System( Orders | Reservations | Orders, Orders | Payments | Orders, ) multiprocess = Multiprocess(system) # Start multiprocessing system. with multiprocess: with Orders() as app: retries = 50 while not app.repository[order_id].is_reserved: time.sleep(0.1) retries -= 1 assert retries, "Failed set order.is_reserved" while retries and not app.repository[order_id].is_paid: time.sleep(0.1) retries -= 1 assert retries, "Failed set order.is_paid"
def setUp(self): # Shutdown base actor system, if running. # ActorSystem().shutdown() # Set environment. set_db_uri() # Define system. self.system = System(Orders | Reservations | Orders | Payments | Orders)
def test_multiprocessing(self): set_db_uri() key1, key2, key3 = uuid4(), uuid4(), uuid4() value1, value2, value3 = 11111, 22222, 33333 self.close_connections_before_forking() pipeline_ids = [1, 2, 3] runner = MultiprocessRunner( system=self.system, pipeline_ids=pipeline_ids, infrastructure_class=self.infrastructure_class) # Start running operating system processes. with runner: # Get local application object. paxosprocess0 = runner.paxosprocess0 assert isinstance(paxosprocess0, PaxosProcess) # Start proposing values on the different system pipelines. paxosprocess0.change_pipeline(1) started1 = datetime.datetime.now() paxosprocess0.propose_value(key1, value1) paxosprocess0.change_pipeline(2) started2 = datetime.datetime.now() paxosprocess0.propose_value(key2, value2) paxosprocess0.change_pipeline(3) started3 = datetime.datetime.now() paxosprocess0.propose_value(key3, value3) # Check all the process applications have expected final values. paxosprocess1 = runner.paxosprocess1 paxosprocess2 = runner.paxosprocess1 assert isinstance(paxosprocess1, PaxosProcess) paxosprocess0.repository.use_cache = False paxosprocess1.repository.use_cache = False paxosprocess2.repository.use_cache = False self.assert_final_value(paxosprocess0, key1, value1) self.assert_final_value(paxosprocess1, key1, value1) self.assert_final_value(paxosprocess2, key1, value1) duration1 = (datetime.datetime.now() - started1).total_seconds() print("Resolved paxos 1 with multiprocessing in %ss" % duration1) self.assert_final_value(paxosprocess0, key2, value2) self.assert_final_value(paxosprocess1, key2, value2) self.assert_final_value(paxosprocess2, key2, value2) duration2 = (datetime.datetime.now() - started2).total_seconds() print("Resolved paxos 2 with multiprocessing in %ss" % duration2) self.assert_final_value(paxosprocess0, key3, value3) self.assert_final_value(paxosprocess1, key3, value3) self.assert_final_value(paxosprocess2, key3, value3) duration3 = (datetime.datetime.now() - started3).total_seconds() print("Resolved paxos 3 with multiprocessing in %ss" % duration3)
def test_multi_threaded(self): if 'TRAVIS_PYTHON_VERSION' in os.environ: if self.infrastructure_class is SQLAlchemyApplication: self.skipTest( "There's an intermittent problem with the multi-threaded" "runner with SQLAlchemy and Python 3.7 on Travis. Fix me :)." ) set_db_uri() key1, key2, key3 = uuid4(), uuid4(), uuid4() value1, value2, value3 = 11111, 22222, 33333 runner = MultiThreadedRunner( system=self.system, infrastructure_class=self.infrastructure_class) with runner: paxosprocess0 = runner.processes['paxosprocess0'] paxosprocess1 = runner.processes['paxosprocess1'] paxosprocess2 = runner.processes['paxosprocess2'] started1 = datetime.datetime.now() assert isinstance(paxosprocess0, PaxosProcess) paxosprocess0.propose_value(key1, value1) ended1 = (datetime.datetime.now() - started1).total_seconds() # Check each process has expected final value. self.assert_final_value(paxosprocess0, key1, value1) self.assert_final_value(paxosprocess1, key1, value1) self.assert_final_value(paxosprocess2, key1, value1) print("Resolved paxos 1 with multi threads in %ss" % ended1) started2 = datetime.datetime.now() paxosprocess1.propose_value(key2, value2) ended2 = (datetime.datetime.now() - started2).total_seconds() # Check each process has a resolution. self.assert_final_value(paxosprocess0, key2, value2) self.assert_final_value(paxosprocess1, key2, value2) self.assert_final_value(paxosprocess2, key2, value2) print("Resolved paxos 2 with multi threads in %ss" % ended2) started3 = datetime.datetime.now() paxosprocess2.propose_value(key3, value3) ended3 = (datetime.datetime.now() - started3).total_seconds() # Check each process has a resolution. self.assert_final_value(paxosprocess0, key3, value3) self.assert_final_value(paxosprocess1, key3, value3) self.assert_final_value(paxosprocess2, key3, value3) print("Resolved paxos 3 with multi threads in %ss" % ended3)
def test_multiprocessing_performance(self): set_db_uri() num_pipelines = 2 pipeline_ids = range(num_pipelines) runner = MultiprocessRunner( system=self.system, pipeline_ids=pipeline_ids, infrastructure_class=self.infrastructure_class, setup_tables=True) num_proposals = 50 self.close_connections_before_forking() with runner: sleep(1) # Construct an application instance in this process. paxos_process = runner.paxosprocess0 assert isinstance(paxos_process, PaxosProcess) # Don't use the cache, so as to keep checking actual database. paxos_process.repository.use_cache = False # Start timing (just for fun). started = datetime.datetime.now() # Propose values. proposals = list(((uuid4(), i) for i in range(num_proposals))) for key, value in proposals: paxos_process.change_pipeline((value % len(pipeline_ids))) print("Proposing key {} value {}".format(key, value)) paxos_process.propose_value(key, str(value)) sleep(0.0) # Check final values. for key, value in proposals: print("Asserting final value for key {} value {}".format( key, value)) self.assert_final_value(paxos_process, key, str(value)) # Print timing information (just for fun). duration = (datetime.datetime.now() - started).total_seconds() print( "Resolved {} paxoses with multiprocessing in {:.4f}s ({:.4f}s each)" .format(num_proposals, duration, duration / num_proposals))
def test_multiprocessing_performance(self): set_db_uri() system = PaxosSystem(setup_tables=True, infrastructure_class=self.infrastructure_class) num_pipelines = 1 pipeline_ids = range(num_pipelines) paxos_process_class = system.process_classes['paxosprocess0'] multiprocess = MultiprocessRunner(system=system, pipeline_ids=pipeline_ids) num_proposals = 50 self.close_connections_before_forking() with multiprocess: sleep(1) paxos_process = system.construct_app( process_class=paxos_process_class, pipeline_id=pipeline_ids[0]) assert isinstance(paxos_process, PaxosProcess) paxos_process.use_cache = False paxos_process.repository._use_cache = False with paxos_process: started = datetime.datetime.now() expectations = list( ((uuid4(), i) for i in range(num_proposals))) for key, value in expectations: paxos_process.change_pipeline((value % len(pipeline_ids))) print("Proposing key {} value {}".format(key, value)) paxos_process.propose_value(key, str(value)) sleep(0.0) for key, value in expectations: print("Asserting final value for key {} value {}".format( key, value)) self.assert_final_value(paxos_process, key, str(value)) duration = (datetime.datetime.now() - started).total_seconds() print( "Resolved {} paxoses with multiprocessing in {:.4f}s ({:.4f}s each)" .format(num_proposals, duration, duration / num_proposals))
def test_multi_threaded(self, is_skipped=True): if is_skipped: self.skipTest( "There's an intermittent problem with the multi-threaded" "runner with SQLAlchemy. Fix me :).") set_db_uri() system = PaxosSystem( setup_tables=True, infrastructure_class=self.infrastructure_class, ) key1, key2, key3 = uuid4(), uuid4(), uuid4() value1, value2, value3 = 11111, 22222, 33333 with MultiThreadedRunner(system): paxosprocess0 = system.processes['paxosprocess0'] paxosprocess1 = system.processes['paxosprocess1'] paxosprocess2 = system.processes['paxosprocess2'] started1 = datetime.datetime.now() assert isinstance(paxosprocess0, PaxosProcess) paxosprocess0.propose_value(key1, value1) ended1 = (datetime.datetime.now() - started1).total_seconds() # Check each process has expected final value. self.assert_final_value(paxosprocess0, key1, value1) self.assert_final_value(paxosprocess1, key1, value1) self.assert_final_value(paxosprocess2, key1, value1) print("Resolved paxos 1 with multi threads in %ss" % ended1) started2 = datetime.datetime.now() paxosprocess1.propose_value(key2, value2) ended2 = (datetime.datetime.now() - started2).total_seconds() # Check each process has a resolution. self.assert_final_value(paxosprocess0, key2, value2) self.assert_final_value(paxosprocess1, key2, value2) self.assert_final_value(paxosprocess2, key2, value2) print("Resolved paxos 2 with multi threads in %ss" % ended2) started3 = datetime.datetime.now() paxosprocess2.propose_value(key3, value3) ended3 = (datetime.datetime.now() - started3).total_seconds() # Check each process has a resolution. self.assert_final_value(paxosprocess0, key3, value3) self.assert_final_value(paxosprocess1, key3, value3) self.assert_final_value(paxosprocess2, key3, value3) print("Resolved paxos 3 with multi threads in %ss" % ended3)
def test_multiprocessing_singleapp_system(self): system = System(Examples | Examples, setup_tables=True) set_db_uri() with Examples() as app, Multiprocess(system): aggregate = ExampleAggregate.__create__() aggregate.__save__() assert aggregate.id in app.repository # Check the aggregate is moved on. retries = 50 while not app.repository[aggregate.id].is_moved_on: time.sleep(0.1) retries -= 1 assert retries, "Failed to move"
def test_multiprocessing(self): set_db_uri() system = PaxosSystem(setup_tables=True, infrastructure_class=self.infrastructure_class) key1, key2, key3 = uuid4(), uuid4(), uuid4() value1, value2, value3 = 11111, 22222, 33333 paxos_process_class = system.process_classes['paxosprocess0'] paxos_process = system.construct_app(paxos_process_class, pipeline_id=2) paxos_process.repository._use_cache = False self.close_connections_before_forking() pipeline_ids = [1, 2, 3] with MultiprocessRunner(system=system, pipeline_ids=pipeline_ids), paxos_process: assert isinstance(paxos_process, PaxosProcess) sleep(1) started1 = datetime.datetime.now() paxos_process.propose_value(key1, value1) paxos_process.change_pipeline(2) started2 = datetime.datetime.now() paxos_process.propose_value(key2, value2) paxos_process.change_pipeline(3) started3 = datetime.datetime.now() paxos_process.propose_value(key3, value3) self.assert_final_value(paxos_process, key1, value1) duration1 = (datetime.datetime.now() - started1).total_seconds() print("Resolved paxos 1 with multiprocessing in %ss" % duration1) self.assert_final_value(paxos_process, key2, value2) duration2 = (datetime.datetime.now() - started2).total_seconds() print("Resolved paxos 2 with multiprocessing in %ss" % duration2) self.assert_final_value(paxos_process, key3, value3) duration3 = (datetime.datetime.now() - started3).total_seconds() print("Resolved paxos 3 with multiprocessing in %ss" % duration3)
def test_multiprocessing(self): set_db_uri() system = PaxosSystem(setup_tables=True, processes=3) with Multiprocess(system, pipeline_ids=[1, 2, 3]), PaxosProcess(name='paxosprocess0', pipeline_id=1) as \ paxosprocess0: sleep(3) print("Running {}".format(datetime.datetime.now())) paxos1 = paxosprocess0.propose_value( 11111, quorum_size=system.quorum_size, assume_leader=True) paxosprocess0.change_pipeline(2) paxos2 = paxosprocess0.propose_value( 22222, quorum_size=system.quorum_size, assume_leader=True) paxosprocess0.change_pipeline(3) paxos3 = paxosprocess0.propose_value( 33333, quorum_size=system.quorum_size, assume_leader=True) # Check each process has a resolution. while True: if paxosprocess0.repository[paxos1.id].resolution: break self.assertEqual( paxosprocess0.repository[paxos1.id].resolution.value, 11111) print("") print("") print("") print("") print("Finished {}".format(datetime.datetime.now())) while True: if paxosprocess0.repository[paxos2.id].resolution: break self.assertEqual( paxosprocess0.repository[paxos2.id].resolution.value, 22222) while True: if paxosprocess0.repository[paxos3.id].resolution: break self.assertEqual( paxosprocess0.repository[paxos3.id].resolution.value, 33333)
def set_db_uri(self): set_db_uri()
def test_multipipeline_multiprocessing_multiapp(self): set_db_uri() system = System((Orders, Reservations, Orders, Payments, Orders), setup_tables=True) num_pipelines = 3 pipeline_ids = range(num_pipelines) multiprocess = Multiprocess(system, pipeline_ids=pipeline_ids) num_orders_per_pipeline = 5 order_ids = [] # Start multiprocessing system. with multiprocess, Orders(setup_table=True) as orders: # Create some new orders. for _ in range(num_orders_per_pipeline): for pipeline_id in pipeline_ids: orders.change_pipeline(pipeline_id) order_id = create_new_order() order_ids.append(order_id) time.sleep(0.05) # Wait for orders to be reserved and paid. retries = 10 + 10 * num_orders_per_pipeline * len(pipeline_ids) for i, order_id in enumerate(order_ids): while not orders.repository[order_id].is_reserved: time.sleep(0.1) retries -= 1 assert retries, "Failed set order.is_reserved {} ({})".format( order_id, i) while retries and not orders.repository[order_id].is_paid: time.sleep(0.1) retries -= 1 assert retries, "Failed set order.is_paid ({})".format(i) # Calculate timings from event timestamps. order_aggregates = [orders.repository[oid] for oid in order_ids] first_timestamp = min([o.__created_on__ for o in order_aggregates]) last_timestamp = max( [o.__last_modified__ for o in order_aggregates]) duration = last_timestamp - first_timestamp rate = len(order_ids) / float(duration) period = 1 / rate print( "Orders system processed {} orders in {:.3f}s at rate of {:.1f} " "orders/s, {:.3f}s each".format(len(order_ids), duration, rate, period)) # Print min, average, max duration. durations = [ o.__last_modified__ - o.__created_on__ for o in order_aggregates ] print("Min order processing time: {:.3f}s".format(min(durations))) print("Mean order processing time: {:.3f}s".format( sum(durations) / len(durations))) print("Max order processing time: {:.3f}s".format(max(durations)))
def setUp(self): # Set environment. set_db_uri() # Define system. self.system = System(Orders | Reservations | Orders | Payments | Orders, infrastructure_class=self.infrastructure_class)