def rcv_getrsp(self, getrsp): seqno = getrsp.msg_id if seqno in self.pending_get_rsp: self.pending_get_rsp[seqno].add( (getrsp.from_node, getrsp.value, getrsp.metadata)) if len(self.pending_get_rsp[seqno]) >= DynamoNode.R: _logger.info("%s: read %d copies of %s=? so done", self, DynamoNode.R, getrsp.key) _logger.debug( " copies at %s", [(node.name, value) for (node, value, _) in self.pending_get_rsp[seqno]]) # Build up all the distinct values/metadata values for the response to the original request results = set([(value, metadata) for (node, value, metadata) in self.pending_get_rsp[seqno]]) # Tidy up tracking data structures original_msg = self.pending_get_msg[seqno] del self.pending_req[GetReq][seqno] del self.pending_get_rsp[seqno] del self.pending_get_msg[seqno] # Reply to the original client, including all received values client_getrsp = ClientGetRsp( original_msg, [value for (value, metadata) in results], [metadata for (value, metadata) in results]) Framework.send_message(client_getrsp) else: pass # Superfluous reply
def test_simple_put(self): for _ in range(6): dynamo1.DynamoNode() a = dynamo1.DynamoClientNode('a') a.put('K1', None, 1) Framework.schedule() print History.ladder()
def get(): if (request.method == 'POST'): key = request.form.get('key') global nodes nodes = [] global retvalue retvalue = "" global vclock vclock = {} global found found = False client = DynamoClientNode('client') client.get(key) Framework.schedule() if (found == False): return '''<H1>Dynamo</H1><br> <H2>Response</H2><br> <H2>Key: {}</H2> <H2>Value: {}</H2> <A href = "http://localhost:5000/put"> Put </A><br> <A href = "http://localhost:5000/get"> Get </A>'''.format( key, "Not found") else: return '''<H1>Dynamo</H1><br> <H2>Response</H2><br> <H2>Key: {}</H2> <H2>Value: {}</H2> <H2>Retrieved from: {}</H2> <H2>Coordinator Node and Vector Clock: {}</H2><br> <A href = "http://localhost:5000/put"> Put </A><br> <A href = "http://localhost:5000/get"> Get </A>'''.format( key, retvalue, nodes, vclock) return '''<H1>Dynamo</H1><br>
def rcv_clientput(self, msg): preference_list, avoided = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes) non_extra_count = DynamoNode.N - len(avoided) # Determine if we are in the list if self not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) metadata = (self.name, seqno) # For now, metadata is just sequence number at coordinator # Send out to preference list, and keep track of who has replied self.pending_req[PutReq][seqno] = set() self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 for ii, node in enumerate(preference_list): if ii >= non_extra_count: # This is an extra node that's only include because of a failed node handoff = avoided else: handoff = None # Send message to get node in preference list to store putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno, handoff=handoff) self.pending_req[PutReq][seqno].add(putmsg) Framework.send_message(putmsg) reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break
def __init__(self, gui): Framework.__init__(self, gui) ground = self.world.create_body() body = self.world.create_static_body( position=(0, -8), allow_sleep=False, shape_fixture=b2d.fixture_def(density=5.0), shapes=[ b2d.polygon_shape((0.5, 10), (10, 0), 0), b2d.polygon_shape((0.5, 10), (-10, 0), 0), b2d.polygon_shape((10, 0.5), (0, 10), 0), b2d.polygon_shape((10, 0.5), (0, -10), 0), ] ) pdef = b2d.particle_system_def(viscous_strength=100,spring_strength=0.0) self.psystem = self.world.create_particle_system(pdef) self.psystem.radius = 0.3 self.psystem.damping = 0.2 pgDef = b2d.particle_group_def(flags=b2d.ParticleFlag.waterParticle, group_flags=b2d.ParticleGroupFlag.solidParticleGroup, strength=0.0) self.group = self.psystem.create_particle_group(pgDef)
def rcv_clientput(self, msg): preference_list = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes)[0] # Determine if we are in the list if self not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) metadata = (self.name, seqno) # For now, metadata is just sequence number at coordinator # Send out to preference list, and keep track of who has replied self.pending_req[PutReq][seqno] = set() self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 for node in preference_list: # Send message to get node in preference list to store putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno) self.pending_req[PutReq][seqno].add(putmsg) Framework.send_message(putmsg) reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break
def rcv_getrsp(self, getrsp): seqno = getrsp.msg_id if seqno in self.pending_get_rsp: self.pending_get_rsp[seqno].add((getrsp.from_node, getrsp.value, getrsp.metadata)) if len(self.pending_get_rsp[seqno]) >= DynamoNode.R: _logger.info("%s: read %d copies of %s=? so done", self, DynamoNode.R, getrsp.key) # _logger.debug(" copies at %s", [(node.name, value) for (node, value, _) in self.pending_get_rsp[seqno]]) # Coalesce all compatible (value, metadata) pairs across the responses results = VectorClock.coalesce2([(value, metadata) for (node, value, metadata) in self.pending_get_rsp[seqno]]) # Tidy up tracking data structures original_msg = self.pending_get_msg[seqno] del self.pending_req[GetReq][seqno] del self.pending_get_rsp[seqno] del self.pending_get_msg[seqno] # Reply to the original client, including all received values client_getrsp = ClientGetRsp(original_msg, [value for (value, metadata) in results], [metadata for (value, metadata) in results]) #modified con = self.connections[self.servers.index(client_getrsp.to_node)] Framework.send_message(client_getrsp, con) ######################################## else: pass # Superfluous reply
def __init__(self, addr, config_file='server_config'): super(DynamoNode, self).__init__() self.local_store = MerkleTree() # key => (value, metadata) self.pending_put_rsp = {} # seqno => set of nodes that have stored self.pending_put_msg = {} # seqno => original client message self.pending_get_rsp = {} # seqno => set of (node, value, metadata) tuples self.pending_get_msg = {} # seqno => original client message # seqno => set of requests sent to other nodes, for each message class self.pending_req = {PutReq: {}, GetReq: {}} self.failed_nodes = [] self.pending_handoffs = {} # Rebuild the consistent hash table self.addr = addr self.servers = [] self.db = leveldb.LevelDB('./' + addr + '_db') f = open(config_file, 'r') for line in f.readlines(): line = line.rstrip() self.servers.append(line) for i, server in enumerate(self.servers): DynamoNode.nodelist.append(server) DynamoNode.chash = ConsistentHashTable(DynamoNode.nodelist, DynamoNode.T) # Run a timer to retry failed nodes #self.pool = gevent.pool.Group() #self.pool.spawn(self.retry_failed_node) Framework.setNodes(DynamoNode.nodelist)
def __init__(self): Framework.__init__(self, 'cli') self._name = Constants.NAME self._prompt_template = '{color_main}{main}{color_module}[{module}]{color_reset} > ' self._base_prompt = self._prompt_template.format(color_main=Colors.C, main='', color_module=Colors.O, module=self._name, color_reset=Colors.N) # Establish dynamic paths for framework elements self.path_app = Framework.path_app = sys.path[0] self.path_core = Framework.path_core = os.path.join( self.path_app, 'core') self.path_libs = Framework.path_libs = os.path.join( self.path_app, 'libs') self.path_modules = Framework.path_modules = os.path.join( self.path_app, 'modules') # Init framework self.options = self._global_options self._init_global_options() self._init_global_vars() self._init_home() self.show_banner() self.do_reload(None) self._history_load()
def rcv_clientput(self, msg): preference_list = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N)[0] # Determine if we are in the list if self not in preference_list: # Forward to the coordinator for this key _logger.info("put(%s=%s) maps to %s", msg.key, msg.value, preference_list) coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: # Use an incrementing local sequence number to distinguish # multiple requests for the same key seqno = self.generate_sequence_number() _logger.info("%s, %d: put %s=%s", self, seqno, msg.key, msg.value) metadata = (self.name, seqno) # For now, metadata is just sequence number at coordinator # Send out to preference list, and keep track of who has replied self.pending_put_rsp[seqno] = set() self.pending_put_msg[seqno] = msg reqcount = 0 for node in preference_list: # Send message to get node in preference list to store putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno) Framework.send_message(putmsg) reqcount = reqcount + 1 if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break
def rcv_clientput(self, msg): preference_list, avoided = DynamoNode.chash.find_nodes( msg.key, DynamoNode.N, self.failed_nodes) avoided = avoided[:DynamoNode.N] non_extra_count = DynamoNode.N - len(avoided) if self not in preference_list: coordinator = preference_list[0] Framework.forward_message(msg, coordinator) else: seqno = self.generate_seq_num() metadata = copy.deepcopy(msg.metadata) metadata.update(self.name, seqno) self.pending_put_rsp = set() self.pending_put_msg = msg reqcount = 0 for i, node in enumerate(preference_list): handoff = avoided if (i >= non_extra_count) else None putmsg = PutReq(self, node, msg.key, msg.value, metadata, msg_id=seqno, handoff=handoff) Framework.send_message(putmsg) reqcount += 1 if reqcount >= DynamoNode.N: break
def retry_failed_node(self, _): # Permanently repeating timer if self.failed_nodes: node = self.failed_nodes.pop(0) # Send a test message to the oldest failed node pingmsg = PingReq(self, node) Framework.send_message(pingmsg) # Restart the timer TimerManager.start_timer(self, reason="retry", priority=15, callback=self.retry_failed_node)
def test_put2_fail_nodes23_5(self): """Show Put after a failure including handoff, and the resulting Pings""" (a, pref_list) = self.put_fail_nodes23(dynamo4) coordinator = pref_list[0] from_line = len(History.history) a.put('K1', None, 2, destnode=coordinator) # Send client request to coordinator for clarity Framework.schedule(timers_to_process=10) print History.ladder(force_include=pref_list, start_line=from_line, spacing=16)
def test_put2_fail_nodes23_2(self): """Show second request for same key skipping failed nodes""" (a, pref_list) = self.put_fail_nodes23(dynamo2) coordinator = pref_list[0] from_line = len(History.history) a.put('K1', None, 2, destnode=coordinator) # Send client request to coordinator for clarity Framework.schedule() print History.ladder(force_include=pref_list, start_line=from_line, spacing=16)
def test_partition_restore_metadata(self): self.partition() self.partition_repair() # Put a new value, which coalesces a = Node.node['a'] getrsp = a.last_msg putmsg = a.put('K1', getrsp.metadata, 101) Framework.schedule(timers_to_process=0) print putmsg.metadata
def __init__(self, source): info("init") Framework.__init__(self) self.output_type = type(BaseMessage) self.input_type = type(BaseMessage) assert issubclass(type(source), Source) assert source.output_type == self.input_type self.next = source
def test_put2_fail_nodes23_3(self): """Show PingReq failing""" (a, pref_list) = self.put_fail_nodes23(dynamo4) coordinator = pref_list[0] a.put('K1', None, 2, destnode=coordinator) # Send client request to coordinator for clarity Framework.schedule(timers_to_process=0) from_line = len(History.history) Framework.schedule(timers_to_process=3) print History.ladder(force_include=pref_list, start_line=from_line, spacing=16)
def step(self, settings): Framework.step(self, settings) r = random.random() px =(random.random() - 0.5)*10.0 py =(random.random() - 0.5)*2.0 if self.step_count < 1500: pd = b2d.particle_def(group=self.group, position=(px, py))#, velocity=(vx,vy)) self.psystem.create_particle(pd)
def retry_failed_node(self, _): if self.failed_nodes: node = self.failed_nodes.pop(0) pingmsg = PingReq(self, node) Framework.send_message(pingmsg) TimeManager.start_timer(self, reason='retry', priority=15, callback=self.retry_failed_node)
def __init__(self, source): Framework.__init__(self) self.output_type = type(BaseMessage) self.input_type = type(BaseMessage) assert issubclass(source, Source) assert source.output_type == self.input_type self.next = source self.iterator_style = True info("using self.iterator_style = %s" % str(self.iterator_style))
def rcv_put(self, putmsg): self.store(putmsg.key, putmsg.value, putmsg.metadata) if putmsg.handoff is not None: for failed_node in putmsg.handoff: self.failed_nodes.append(failed_node) if failed_node not in self.pending_handoffs: self.pending_handoffs[failed_node].add(putmsg.key) putrsp = PutRsp(putmsg) Framework.send_message(putrsp)
def partition_repair(self): # Repair the partition History.add("announce", "Repair network partition") Framework.cuts = [] Framework.schedule(timers_to_process=12) # Get from node a a = Node.node['a'] a.get('K1') Framework.schedule(timers_to_process=0)
def test_double_put(self): for _ in range(6): dynamo1.DynamoNode() a = dynamo1.DynamoClientNode('a') b = dynamo1.DynamoClientNode('b') a.put('K1', None, 1) Framework.schedule(1) b.put('K2', None, 17) Framework.schedule() print History.ladder(spacing=14)
def rcv_pingrsp(self, pingmsg): revocered_node = pingmsg.from_node while recovered_node in self.failed_nodes: self.failed_nodes.remove(recovered_node) if recovered_node in self.pending_handoffs: for key in self.pending_handoffs[recovered_node]: value, metadata = self.retrieve(key) putmsg = PutReq(self, recovered_node, key, value, metadata) Framework.send_message(putmsg) del self.pending_handoffs[recovered_node]
def put(self, key, metadata, value, destnode=None): if destnode is None: destnode = random.choice(DynamoNode.nodelist) if len(metadata) == 1 and metadata[0] is None: metadata = VectorClock() else: metadata = VectorClock.converge(metadata) putmsg = ClientPut(self, destnode, key, value, metadata) Framework.send_message(putmsg) return putmsg
def rcv_putrsp(self, putrsp): seqno = putrsp.msg_id if seqno in self.pending_put_rsp: self.pending_put_rsp[seqno].add(putrsp.from_node) orig_msg = self.pending_put_msg[seqno] del self.pending_req[PutReq][seqno] del self.pending_put_rsp[seqno] del self.pending_put_msg[seqno] client_putrsp = ClientPutRsp(orig_msg) Framework.send_message(client_putrsp)
def put_fail_initial_node(self, cls): for _ in range(6): cls.DynamoNode() a = cls.DynamoClientNode('a') destnode = random.choice(cls.DynamoNode.nodelist) a.put('K1', None, 1, destnode=destnode) # Fail at the forwarding node before it gets a chance to forward destnode.fail() Framework.schedule() print History.ladder()
def test_simple_get(self): for _ in range(6): dynamo1.DynamoNode() a = dynamo1.DynamoClientNode('a') a.put('K1', None, 1) Framework.schedule() from_line = len(History.history) a.get('K1') Framework.schedule() print History.ladder(start_line=from_line)
def __init__(self): Framework.__init__(self) world = self.world ground = world.create_static_body(shapes=b2.Edge((-20, 0),( 20, 0))) controller = world.create_buoyancy_controller( offset=15, normal=(0, 1), density=2, linear_drag=2, angular_drag=1) # Create a few spheres to bob around for i in range(7): body = self.world.create_dynamic_body( position=(-10+4.0*i, 20), fixtures=b2.Fixture(shape=b2.Circle(radius=1.0), density=1.0) ) controller.add_body(body) # Create a bridge, and add it to the controller num_planks = 30 plank_bodies = create_bridge(self.world, ground, (1.0, 0.25), (-14.5, 5), num_planks, 0.2, 1) for body in plank_bodies: controller.add_body(body) # Create a circle underneath the bridge body = self.world.create_dynamic_body( position=(-10.0, 0), fixtures=b2.Fixture(shape=b2.Circle(radius=1.0), density=1.0) ) controller.add_body(body) # And finally, some triangles for i in range(5): body = self.world.create_dynamic_body( position=(-10+3.0*i, 20), fixtures=b2.Fixture( shape=b2.Polygon( vertices=[(-0.5,0),(0,-0.5),(0.5, 0.0)]), density=1.0) ) controller.add_body(body) # And (really) finally this time, just something so we can be sure # edges work, too. edge = world.create_dynamic_body( fixtures=b2.Fixture(shape=b2.Edge((5, 0),(5, 3)), density=1.0) ) controller.add_body(edge)
def test_put2_fail_nodes23_5(self): """Show Put after a failure including handoff, and the resulting Pings""" (a, pref_list) = self.put_fail_nodes23(dynamo4) coordinator = pref_list[0] from_line = len(History.history) a.put('K1', None, 2, destnode=coordinator ) # Send client request to coordinator for clarity Framework.schedule(timers_to_process=10) print History.ladder(force_include=pref_list, start_line=from_line, spacing=16)
def rcv_get(self, getmsg): _logger.info("%s: retrieve %s=?", self, getmsg.key) (value, metadata) = self.retrieve(getmsg.key) getrsp = GetRsp(getmsg, value, metadata) global nodes nodes.append(self.name) global retvalue retvalue = value global vclock vclock = metadata Framework.send_message(getrsp)
def test_put2_fail_nodes23_2(self): """Show second request for same key skipping failed nodes""" (a, pref_list) = self.put_fail_nodes23(dynamo2) coordinator = pref_list[0] from_line = len(History.history) a.put('K1', None, 2, destnode=coordinator ) # Send client request to coordinator for clarity Framework.schedule() print History.ladder(force_include=pref_list, start_line=from_line, spacing=16)
def rcv_put(self, putmsg): _logger.info("%s: store %s=%s", self, putmsg.key, putmsg.value) self.store(putmsg.key, putmsg.value, putmsg.metadata) if putmsg.handoff is not None: for failed_node in putmsg.handoff: self.failed_nodes.append(failed_node) if failed_node not in self.pending_handoffs: self.pending_handoffs[failed_node] = set() self.pending_handoffs[failed_node].add(putmsg.key) putrsp = PutRsp(putmsg) Framework.send_message(putrsp)
def test_put2_fail_nodes23_6(self): """Show hinted handoff after recovery""" (a, pref_list) = self.put_fail_nodes23(dynamo4) coordinator = pref_list[0] a.put('K1', None, 2, destnode=coordinator) # Send client request to coordinator for clarity Framework.schedule(timers_to_process=10) from_line = len(History.history) pref_list[1].recover() pref_list[2].recover() Framework.schedule(timers_to_process=15) print History.ladder(force_include=pref_list, start_line=from_line, spacing=16)
def run(): framework = Framework() framework.command = "ls" framework.numRuns = 3 framework.params = [('-l', ['']), ('-a', [''])] framework.runBenchmarks() framework.ppOutput()
def run(): framework = Framework() framework.command = "cat" framework.numRuns = 3 framework.params = [('', ['ls_test.py', 'cat_test.py', 'framework.py'])] framework.runBenchmarks() framework.ppOutput()
def rcv_pingrsp(self, pingmsg): # Remove all instances of recovered node from failed node list recovered_node = pingmsg.from_node while recovered_node in self.failed_nodes: self.failed_nodes.remove(recovered_node) if recovered_node in self.pending_handoffs: for key in self.pending_handoffs[recovered_node]: # Send our latest value for this key (value, metadata) = self.retrieve(key) putmsg = PutReq(self, recovered_node, key, value, metadata) Framework.send_message(putmsg) del self.pending_handoffs[recovered_node]
def test_put2_fail_nodes23_3(self): """Show PingReq failing""" (a, pref_list) = self.put_fail_nodes23(dynamo4) coordinator = pref_list[0] a.put('K1', None, 2, destnode=coordinator ) # Send client request to coordinator for clarity Framework.schedule(timers_to_process=0) from_line = len(History.history) Framework.schedule(timers_to_process=3) print History.ladder(force_include=pref_list, start_line=from_line, spacing=16)
def main(): for i in range(1, case_cnt): try: case_name = "case%d" % i f = Framework(conf_file, case_name) f.build() if not f.after_build(): raise RuntimeError("Framework build Fail!") print(f) except Exception as e: print(e.__repr__() + ", Init Failed!") break
def put_message(self, fromnode, key, value, metadata): #print 'client put!!!' metadata = pickle.loads(metadata) if metadata is None: metadata = VectorClock() else: # A Put operation always implies convergence metadata = VectorClock.converge(metadata) putmsg = ClientPut(fromnode, self.addr, key, value, metadata) Framework.send_message(putmsg) # Framework.schedule(timers_to_process=0) Framework.schedule()
def initalizeEnvironment(environment, logger): if environment != '': # Initialize the db db = Database(DB_NAME, DB_HOST, DB_PORT) # Initialize simple fog datacenter ''' Can be SimpleFog, BitbrainFog, AzureFog // Datacenter ''' if environment != '': datacenter = Datacenter(HOSTS_IP, environment, 'Virtual') else: datacenter = AzureFog(HOSTS) # Initialize workload ''' Can be SWSD, BWGD, BWGD2 // DFW ''' if environment != '': workload = DFW(NEW_CONTAINERS, 1.5, db) else: workload = BWGD2(NEW_CONTAINERS, 1.5) # Initialize scheduler ''' Can be LRMMTR, RF, RL, RM, Random, RLRMMTR, TMCR, TMMR, TMMTR, GA, GOBI (arg = 'energy_latency_'+str(HOSTS)) ''' scheduler = GOBIScheduler( 'energy_latency_' + str(HOSTS)) # GOBIScheduler('energy_latency_'+str(HOSTS)) # Initialize Environment hostlist = datacenter.generateHosts() if environment != '': env = Framework(scheduler, CONTAINERS, INTERVAL_TIME, hostlist, db, environment, logger) else: env = Simulator(TOTAL_POWER, ROUTER_BW, scheduler, CONTAINERS, INTERVAL_TIME, hostlist) # Execute first step newcontainerinfos = workload.generateNewContainers( env.interval) # New containers info deployed = env.addContainersInit( newcontainerinfos) # Deploy new containers and get container IDs start = time() decision = scheduler.placement( deployed) # Decide placement using container ids schedulingTime = time() - start migrations = env.allocateInit(decision) # Schedule containers workload.updateDeployedContainers(env.getCreationIDs( migrations, deployed)) # Update workload allocated using creation IDs print("Deployed containers' creation IDs:", env.getCreationIDs(migrations, deployed)) print("Containers in host:", env.getContainersInHosts()) print("Schedule:", env.getActiveContainerList()) printDecisionAndMigrations(decision, migrations) # Initialize stats stats = Stats(env, workload, datacenter, scheduler) stats.saveStats(deployed, migrations, [], deployed, decision, schedulingTime) return datacenter, workload, scheduler, env, stats
def put_fail_nodes23(self, cls): # Set up 6 nodes and 1 client node for _ in range(6): cls.DynamoNode() a = cls.DynamoClientNode('a') # Fail the second and third node in the preference list pref_list = cls.DynamoNode.chash.find_nodes('K1', 5)[0] a.put('K1', None, 1, destnode=pref_list[0]) Framework.schedule(1) pref_list[1].fail() pref_list[2].fail() Framework.schedule(timers_to_process=2) return a, pref_list
def step(self, settings): Framework.step(self, settings) #print(self.count) self.count -= 1 if self.count <= 0: return self.world.create_dynamic_body( position=(0, 10), allow_sleep=False, fixtures=fixture_def( density=1.0, shape=polygon_shape(box=(0.125, 0.125))), )
def test_partition_restore(self): dynamomessages._show_metadata = True all_nodes = self.partition() self.partition_repair() from_line = len(History.history) # Put a new value, which coalesces a = Node.node['a'] getrsp = a.last_msg a.put('K1', getrsp.metadata, 101) Framework.schedule(timers_to_process=0) # Display, tweaking ordering of nodes so partition is in the middle print History.ladder(force_include=all_nodes, start_line=from_line, spacing=16, key=lambda x: ' ' if x.name == 'b' else x.name) dynamomessages._show_metadata = False
def rcv_putrsp(self, putrsp): seqno = putrsp.msg_id if seqno in self.pending_put_rsp: self.pending_put_rsp[seqno].add(putrsp.from_node) if len(self.pending_put_rsp[seqno]) >= DynamoNode.W: _logger.info("%s: written %d copies of %s=%s so done", self, DynamoNode.W, putrsp.key, putrsp.value) _logger.debug(" copies at %s", [node.name for node in self.pending_put_rsp[seqno]]) # Tidy up tracking data structures original_msg = self.pending_put_msg[seqno] del self.pending_put_rsp[seqno] del self.pending_put_msg[seqno] # Reply to the original client client_putrsp = ClientPutRsp(original_msg) Framework.send_message(client_putrsp) else: pass # Superfluous reply
def retry_request(self, reqmsg): if not isinstance(reqmsg, DynamoRequestMessage): return # Send the request to an additional node by regenerating the preference list preference_list = DynamoNode.chash.find_nodes(reqmsg.key, DynamoNode.N, self.failed_nodes)[0] kls = reqmsg.__class__ # Check the pending-request list for this type of request message if kls in self.pending_req and reqmsg.msg_id in self.pending_req[kls]: for node in preference_list: if node not in [req.to_node for req in self.pending_req[kls][reqmsg.msg_id]]: # Found a node on the new preference list that hasn't been sent the request. # Send it a copy newreqmsg = copy.copy(reqmsg) newreqmsg.to_node = node self.pending_req[kls][reqmsg.msg_id].add(newreqmsg) Framework.send_message(newreqmsg)
def put(self, key, metadata, value, destnode=None): # print "node: ",len(DynamoNode.nodelist) #modified temp = metadata while True: ################################### metadata = temp if destnode is None: # Pick a random node to send the request to destnode = random.choice(DynamoNode.nodelist) # Input metadata is always a sequence, but we always need to insert a # single VectorClock object into the ClientPut message # print '-------------------------choice:', destnode if len(metadata) == 1 and metadata[0] is None: metadata = VectorClock() else: # A Put operation always implies convergence metadata = VectorClock.converge(metadata) putmsg = ClientPut(self.addr, destnode, key, value, metadata) #modified con = self.connections[self.servers.index(destnode)] result = Framework.send_message(putmsg, con) if result is not False: break destnode = None ################################## return result
def __init__(self, addr, config_file='server_config'): super(DynamoNode, self).__init__() self.framework = Framework() self.m_addr = "127.0.0.1:29009" self.local_store = MerkleTree() # key => (value, metadata) self.pending_put_rsp = {} # seqno => set of nodes that have stored self.pending_put_msg = {} # seqno => original client message self.pending_get_rsp = {} # seqno => set of (node, value, metadata) tuples self.pending_get_msg = {} # seqno => original client message # seqno => set of requests sent to other nodes, for each message class self.pending_req = {PutReq: {}, GetReq: {}} self.failed_nodes = [] self.pending_handoffs = {} self.result = {} self.reduceResult = {} self.MapReduceDB = leveldb.LevelDB('./' + addr + '_mrdb') self.mapDict = {} # Rebuild the consistent hash table self.addr = addr self.servers = [] self.db = leveldb.LevelDB('./' + addr + '_db') c = zerorpc.Client(timeout=3) c.connect('tcp://' + self.m_addr) try: c.add_node(self.addr) c.close() except: pass self.pool = gevent.pool.Group() self.pool.spawn(self.retry_failed_node) self.framework.setDynamo(self)
def __init__(self): Framework.__init__(self) world = self.world # Turn off normal gravity world.gravity = (0, 0) ground = world.create_static_body(shapes=b2.Edge((-20, 0),( 20, 0))) controller = world.create_gravity_controller(G=0.8, inv_sqr=True) # Create a few spheres to bob around for i in range(1,4): body = self.world.create_dynamic_body( position=(0.25 * i, 2.0 + 7.5 * i), fixtures=b2.Fixture(shape=b2.Circle(radius=0.25 * i), density=1.0), bullet=True, ) controller.add_body(body) # Create a bridge, and add it to the controller num_planks = 30 plank_bodies = create_bridge(self.world, ground, (1.0, 0.25), (-14.5, 5), num_planks, 0.2, 10) for body in plank_bodies: controller.add_body(body) # Create a circle underneath the bridge body = self.world.create_dynamic_body( position=(-10.0, 0), fixtures=b2.Fixture(shape=b2.Circle(radius=1.0), density=10.0) ) controller.add_body(body) # And finally, some triangles for i in range(5): body = self.world.create_dynamic_body( position=(-10+3.0*i, 20), fixtures=b2.Fixture(shape=b2.Polygon(vertices=[(-0.5,0),(0,-0.5),(0.5, 0.0)]), density=1.0) ) controller.add_body(body)
def rcv_clientget(self, msg): preference_list = DynamoNode.chash.find_nodes(msg.key, DynamoNode.N, self.failed_nodes)[0] # Determine if we are in the list #modified if self.addr not in preference_list: ################################ # Forward to the coordinator for this key _logger.info("get(%s=?) maps to %s", msg.key, preference_list) for e in preference_list: con = self.connections[self.servers.index(e)] result = Framework.forward_message(msg, con, e) if result is not False: break if e not in self.failed_nodes: self.failed_nodes.append(e) return result else: seqno = self.generate_sequence_number() self.pending_req[GetReq][seqno] = set() self.pending_get_rsp[seqno] = set() self.pending_get_msg[seqno] = msg reqcount = 0 #modified value = [] ################ for node in preference_list: getmsg = GetReq(self.addr, node, msg.key, msg_id=seqno) self.pending_req[GetReq][seqno].add(getmsg) #modified con = self.connections[self.servers.index(getmsg.to_node)] result = Framework.send_message(getmsg, con) if result is not False: value.append(result) reqcount = reqcount + 1 ############################################ if reqcount >= DynamoNode.N: # preference_list may have more than N entries to allow for failed nodes break # print value #modified # no value for this key if len(value) < 1: return False ########################## return value