def setUp(self): # Start Plasma store. plasma_store_name, self.p = plasma.start_plasma_store(use_valgrind=USE_VALGRIND) # Connect to Plasma. self.plasma_client = plasma.PlasmaClient(plasma_store_name, None, 64) # For the eviction test self.plasma_client2 = plasma.PlasmaClient(plasma_store_name, None, 0)
def setUp(self): # Start two PlasmaStores. plasma_store_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../build/plasma_store") store_name1 = "/tmp/store{}".format(random.randint(0, 10000)) store_name2 = "/tmp/store{}".format(random.randint(0, 10000)) self.p2 = subprocess.Popen( [plasma_store_executable, "-s", store_name1]) self.p3 = subprocess.Popen( [plasma_store_executable, "-s", store_name2]) # Start two PlasmaManagers. self.port1 = random.randint(10000, 50000) self.port2 = random.randint(10000, 50000) plasma_manager_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../build/plasma_manager") self.p4 = subprocess.Popen([ plasma_manager_executable, "-s", store_name1, "-m", "127.0.0.1", "-p", str(self.port1) ]) self.p5 = subprocess.Popen([ plasma_manager_executable, "-s", store_name2, "-m", "127.0.0.1", "-p", str(self.port2) ]) time.sleep(0.1) # Connect two PlasmaClients. self.client1 = plasma.PlasmaClient(store_name1, "127.0.0.1", self.port1) self.client2 = plasma.PlasmaClient(store_name2, "127.0.0.1", self.port2) time.sleep(0.5)
def setUp(self): # Start two PlasmaStores. plasma_store_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../build/plasma_store") store_name1 = "/tmp/store{}".format(random.randint(0, 10000)) store_name2 = "/tmp/store{}".format(random.randint(0, 10000)) plasma_store_command1 = [plasma_store_executable, "-s", store_name1] plasma_store_command2 = [plasma_store_executable, "-s", store_name2] if USE_VALGRIND: self.p2 = subprocess.Popen( ["valgrind", "--track-origins=yes", "--error-exitcode=1"] + plasma_store_command1) self.p3 = subprocess.Popen( ["valgrind", "--track-origins=yes", "--error-exitcode=1"] + plasma_store_command2) else: self.p2 = subprocess.Popen(plasma_store_command1) self.p3 = subprocess.Popen(plasma_store_command2) # Start two PlasmaManagers. self.port1 = random.randint(10000, 50000) self.port2 = random.randint(10000, 50000) plasma_manager_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../build/plasma_manager") plasma_manager_command1 = [ plasma_manager_executable, "-s", store_name1, "-m", "127.0.0.1", "-p", str(self.port1) ] plasma_manager_command2 = [ plasma_manager_executable, "-s", store_name2, "-m", "127.0.0.1", "-p", str(self.port2) ] if USE_VALGRIND: self.p4 = subprocess.Popen( ["valgrind", "--track-origins=yes", "--error-exitcode=1"] + plasma_manager_command1) self.p5 = subprocess.Popen( ["valgrind", "--track-origins=yes", "--error-exitcode=1"] + plasma_manager_command2) time.sleep(2.0) else: self.p4 = subprocess.Popen(plasma_manager_command1) self.p5 = subprocess.Popen(plasma_manager_command2) time.sleep(0.1) # Connect two PlasmaClients. self.client1 = plasma.PlasmaClient(store_name1, "127.0.0.1", self.port1) self.client2 = plasma.PlasmaClient(store_name2, "127.0.0.1", self.port2) time.sleep(0.5)
def test_delayed_start(self): num_objects = 10 # Create some objects using one client. object_ids = [random_object_id() for _ in range(num_objects)] for i in range(10): create_object_with_id(self.client, object_ids[i], 2000, 2000) # Wait until the objects have been sealed in the store. ready, waiting = self.client.wait(object_ids, num_returns=num_objects) self.assertEqual(set(ready), set(object_ids)) self.assertEqual(waiting, []) # Start a second plasma manager attached to the same store. manager_name, self.p5, self.port2 = plasma.start_plasma_manager(self.store_name, self.redis_address, use_valgrind=USE_VALGRIND) self.processes_to_kill.append(self.p5) # Check that the second manager knows about existing objects. client2 = plasma.PlasmaClient(self.store_name, manager_name) ready, waiting = [], object_ids while True: ready, waiting = client2.wait(object_ids, num_returns=num_objects, timeout=0) if len(ready) == len(object_ids): break self.assertEqual(set(ready), set(object_ids)) self.assertEqual(waiting, [])
def setUp(self): # Start two PlasmaStores. store_name1, self.p2 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND) store_name2, self.p3 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND) # Start a Redis server. redis_address = services.start_redis("127.0.0.1") # Start two PlasmaManagers. manager_name1, self.p4, self.port1 = plasma.start_plasma_manager(store_name1, redis_address, use_valgrind=USE_VALGRIND) manager_name2, self.p5, self.port2 = plasma.start_plasma_manager(store_name2, redis_address, use_valgrind=USE_VALGRIND) # Connect two PlasmaClients. self.client1 = plasma.PlasmaClient(store_name1, manager_name1) self.client2 = plasma.PlasmaClient(store_name2, manager_name2) # Store the processes that will be explicitly killed during tearDown so # that a test case can remove ones that will be killed during the test. # NOTE: If this specific order is changed, valgrind will fail. self.processes_to_kill = [self.p4, self.p5, self.p2, self.p3]
def setUp(self): # Start Plasma. plasma_store_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../build/plasma_store") store_name = "/tmp/store{}".format(random.randint(0, 10000)) self.p = subprocess.Popen([plasma_store_executable, "-s", store_name]) # Connect to Plasma. self.plasma_client = plasma.PlasmaClient(store_name)
def setUp(self): # Start Plasma store. plasma_store_name, self.p1 = plasma.start_plasma_store() self.plasma_client = plasma.PlasmaClient(plasma_store_name) # Start a local scheduler. scheduler_name, self.p2 = photon.start_local_scheduler( plasma_store_name, use_valgrind=USE_VALGRIND) # Connect to the scheduler. self.photon_client = photon.PhotonClient(scheduler_name)
def setUp(self): # Start one Redis server and N pairs of (plasma, photon) redis_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../../core/src/common/thirdparty/redis/src/redis-server") redis_module = os.path.join( os.path.dirname(os.path.abspath(__file__)), "../../core/src/common/redis_module/libray_redis_module.so") assert os.path.isfile(redis_path) assert os.path.isfile(redis_module) node_ip_address = "127.0.0.1" redis_port = new_port() redis_address = "{}:{}".format(node_ip_address, redis_port) self.redis_process = subprocess.Popen([ redis_path, "--port", str(redis_port), "--loglevel", "warning", "--loadmodule", redis_module ]) time.sleep(0.1) # Create a Redis client. self.redis_client = redis.StrictRedis(host=node_ip_address, port=redis_port) # Start one global scheduler. self.p1 = global_scheduler.start_global_scheduler( redis_address, use_valgrind=USE_VALGRIND) self.plasma_store_pids = [] self.plasma_manager_pids = [] self.local_scheduler_pids = [] self.plasma_clients = [] self.photon_clients = [] for i in range(NUM_CLUSTER_NODES): # Start the Plasma store. Plasma store name is randomly generated. plasma_store_name, p2 = plasma.start_plasma_store() self.plasma_store_pids.append(p2) # Start the Plasma manager. # Assumption: Plasma manager name and port are randomly generated by the plasma module. plasma_manager_name, p3, plasma_manager_port = plasma.start_plasma_manager( plasma_store_name, redis_address) self.plasma_manager_pids.append(p3) plasma_address = "{}:{}".format(node_ip_address, plasma_manager_port) plasma_client = plasma.PlasmaClient(plasma_store_name, plasma_manager_name) self.plasma_clients.append(plasma_client) # Start the local scheduler. local_scheduler_name, p4 = photon.start_local_scheduler( plasma_store_name, plasma_manager_name=plasma_manager_name, plasma_address=plasma_address, redis_address=redis_address, static_resource_list=[10, 0]) # Connect to the scheduler. photon_client = photon.PhotonClient(local_scheduler_name, NIL_ACTOR_ID) self.photon_clients.append(photon_client) self.local_scheduler_pids.append(p4)
def setUp(self): # Start two PlasmaStores. store_name1, self.p2 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND) store_name2, self.p3 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND) # Start a Redis server. redis_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "../../common/thirdparty/redis/src/redis-server") redis_port = 6379 with open(os.devnull, "w") as FNULL: self.redis_process = subprocess.Popen([redis_path, "--port", str(redis_port)], stdout=FNULL) time.sleep(0.1) # Start two PlasmaManagers. redis_address = "{}:{}".format("127.0.0.1", redis_port) manager_name1, self.p4, self.port1 = plasma.start_plasma_manager(store_name1, redis_address, use_valgrind=USE_VALGRIND) manager_name2, self.p5, self.port2 = plasma.start_plasma_manager(store_name2, redis_address, use_valgrind=USE_VALGRIND) # Connect two PlasmaClients. self.client1 = plasma.PlasmaClient(store_name1, manager_name1) self.client2 = plasma.PlasmaClient(store_name2, manager_name2)
def setUp(self): # Start Plasma. plasma_store_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../build/plasma_store") store_name = "/tmp/store{}".format(random.randint(0, 10000)) command = [plasma_store_executable, "-s", store_name] if USE_VALGRIND: self.p = subprocess.Popen( ["valgrind", "--track-origins=yes", "--leak-check=full"] + command) time.sleep(2.0) else: self.p = subprocess.Popen(command) # Connect to Plasma. self.plasma_client = plasma.PlasmaClient(store_name)
def setUp(self): # Start a Plasma store. self.store_name, self.p2 = plasma.start_plasma_store(use_valgrind=USE_VALGRIND) # Start a Redis server. self.redis_address = services.start_redis("127.0.0.1") # Start a PlasmaManagers. manager_name, self.p3, self.port1 = plasma.start_plasma_manager( self.store_name, self.redis_address, use_valgrind=USE_VALGRIND) # Connect a PlasmaClient. self.client = plasma.PlasmaClient(self.store_name, manager_name) # Store the processes that will be explicitly killed during tearDown so # that a test case can remove ones that will be killed during the test. self.processes_to_kill = [self.p2, self.p3]
def TEST_GetBeforeAfterPut(numobj_getperf, numobj_put): with eventstats.BenchmarkLogSpan("get_before_after_put", {"numobj_getperf" : numobj_getperf, "numobj_put" : numobj_put}): lst1,putdt1 = plasma_create_objects(numobj_getperf) dt1 = plasma_get_objects(lst1) lst2,putdt2 = plasma_create_objects(numobj_put) #dt2 = plasma_get_objects(lst2) #known bug: this causes a crash #now try to match dt1 get time again dt2 = plasma_get_objects(lst1) lst3,putdt3 = plasma_create_objects(numobj_getperf) dt3 = plasma_get_objects(lst3) print("relative time={} to get initial numobj={} objects".format(dt2/dt1, numobj_getperf)) print("relative time={} to get new numobj={} objects".format(dt3/dt1, numobj_getperf)) if __name__ == "__main__": bench_env = raybench.Env() out = bench_env.ray_init() plasma_socket = out['object_store_addresses'][0].name client = plasma.PlasmaClient(plasma_socket) # TODO: just run one of these TEST_PutAfterPut(10**6) TEST_GetBeforeAfterPut(10**5, 10**6) TEST_PutGetLinearScale() TEST_PutLinearScale() print("BENCHMARK_STATS:", json.dumps({ "config": { }, "events" : eventstats.log_span_events() })) ray.flush_log()
def setUp(self): # Start two PlasmaStores. plasma_store_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../build/plasma_store") store_name1 = "/tmp/store{}".format(random.randint(0, 10000)) store_name2 = "/tmp/store{}".format(random.randint(0, 10000)) plasma_store_command1 = [plasma_store_executable, "-s", store_name1] plasma_store_command2 = [plasma_store_executable, "-s", store_name2] if USE_VALGRIND: self.p2 = subprocess.Popen([ "valgrind", "--track-origins=yes", "--leak-check=full", "--error-exitcode=1" ] + plasma_store_command1) self.p3 = subprocess.Popen([ "valgrind", "--track-origins=yes", "--leak-check=full", "--error-exitcode=1" ] + plasma_store_command2) else: self.p2 = subprocess.Popen(plasma_store_command1) self.p3 = subprocess.Popen(plasma_store_command2) # Start a Redis server. redis_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../common/thirdparty/redis-3.2.3/src/redis-server") self.redis_process = None manager_redis_args = [] if os.path.exists(redis_path): redis_port = 6379 with open(os.devnull, 'w') as FNULL: self.redis_process = subprocess.Popen( [redis_path, "--port", str(redis_port)], stdout=FNULL) time.sleep(0.1) manager_redis_args = [ "-d", "{addr}:{port}".format(addr="127.0.0.1", port=redis_port) ] # Start two PlasmaManagers. self.port1 = random.randint(10000, 50000) self.port2 = random.randint(10000, 50000) plasma_manager_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), "../build/plasma_manager") plasma_manager_command1 = [ plasma_manager_executable, "-s", store_name1, "-m", "127.0.0.1", "-p", str(self.port1) ] + manager_redis_args plasma_manager_command2 = [ plasma_manager_executable, "-s", store_name2, "-m", "127.0.0.1", "-p", str(self.port2) ] + manager_redis_args if USE_VALGRIND: self.p4 = subprocess.Popen([ "valgrind", "--track-origins=yes", "--leak-check=full", "--error-exitcode=1" ] + plasma_manager_command1) self.p5 = subprocess.Popen([ "valgrind", "--track-origins=yes", "--leak-check=full", "--error-exitcode=1" ] + plasma_manager_command2) else: self.p4 = subprocess.Popen(plasma_manager_command1) self.p5 = subprocess.Popen(plasma_manager_command2) # Connect two PlasmaClients. self.client1 = plasma.PlasmaClient(store_name1, "127.0.0.1", self.port1) self.client2 = plasma.PlasmaClient(store_name2, "127.0.0.1", self.port2)