def heavy_load(self): ''' This routine returns if this node is heavily loaded. @return: True if heavy load, False otherwise @rtype: Boolean ''' my_load = DPSNodeDomainStatistics.load_array(self.domain_statistics.values()) message = 'Node %s, Load %s'%(self.location.show_ip(), my_load) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) if my_load > DPSNodeStatValues.Heavy_Load: return (True, my_load) else: return (False, my_load)
def heavy_load(self): ''' This routine returns if this node is heavily loaded. @return: True if heavy load, False otherwise @rtype: Boolean ''' my_load = DPSNodeDomainStatistics.load_array( self.domain_statistics.values()) message = 'Node %s, Load %s' % (self.location.show_ip(), my_load) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) if my_load > DPSNodeStatValues.Heavy_Load: return (True, my_load) else: return (False, my_load)
def load_balance_domain(node_high, node_low): ''' This routine determines which domain can be moved from node1 to node2 or vice-versa that will minimize the load differential between the two i.e. minimize |load(node_high) - load(node_low)| @param node_high: The Higher Loaded Node @type node_high: DPSNodeStatistics @param node_low: The Lower Loaded Node @type node_low: DPSNodeStatistics @return: domain_id @rtype: Integer @raise: Exception if node_high is already lower than node_low or no domain is found ''' #print 'load_balance_domain: Enter High %s, Low %s\r'%(node_high.location.show_ip(), # node_low.location.show_ip()) load_high = DPSNodeDomainStatistics.load_array(node_high.domain_statistics.values()) load_low = DPSNodeDomainStatistics.load_array(node_low.domain_statistics.values()) #print 'load_balance_domain: load_high %s, load_low %s\r'%(load_high, load_low) if load_high <= load_low: raise Exception('Load on Node %s[%.2f], already lower than Node %s[%.2f]'%(node_high.location.show_ip(), load_high, node_low.location.show_ip(), load_low)) domain_id = None domain_ids = {} for node_domain in node_high.domain_statistics.values(): #Check if this domain is present in the lower node domain_id = node_domain.domain_id try: node_domain_low = node_low.domain_statistics[domain_id] continue except Exception: pass #Determine the Load if this domain moved from node_high to node_low load_domain = node_domain.load() #print 'Loop: domain %s, load %s\r'%(node_domain.domain_id, load_domain) if load_domain > load_high: message = 'Coding Error? Load on domain %s[%.2f] on node %s higher the load on node %.2f'%(node_domain.domain_id, load_domain, node_high.location.show_ip(), load_high, ) dcslib.dps_cluster_write_log(DpsLogLevels.WARNING, message) continue load_high_new = load_high - load_domain load_low_new = load_low + load_domain load_diff_new = abs(load_high_new - load_low_new) domain_ids[node_domain] = load_diff_new if domain_id is None: raise Exception('No suitable domain found') #Sort the List based on values domain_tuples = sorted([(value,key) for (key,value) in domain_ids.items()]) #Only chose the 10 differential lowest domains i.e. the domains that cause max swing in load diff domain_tuples = domain_tuples[:10] print 'domain_tuples: %s\r'%domain_tuples domain_list = [] load_high_new = load_high load_low_new = load_low for domain_tuple in domain_tuples: node_domain = domain_tuple[1] domain_id = node_domain.domain_id load_domain = node_domain.load() load_high_new -= load_domain load_low_new += load_domain if load_low_new > DPSNodeStatValues.Heavy_Load: break if load_high_new < load_low_new: break domain_list.append(domain_id) #print 'load_balance_domain: Exit domain_id %s\r'%domain_id if len(domain_list) == 0: raise Exception('No suitable domain found') print 'domain_list: %s\r'%domain_list return domain_list
def mass_transfer_finish(self, dps_location, fcomplete, weight): ''' This routine will be called by the Mass Transfer Callback when mass transfer of data to remote node is done. @param dps_location: The location of the remote node @type dps_location: IPAddressLocation @param fcomplete: Indicates if the transfer was complete or had to be cancelled due to some issue @type fcomplete: False @param weight: The weight of the mass transfer @type weight: Integer ''' message = 'Domain %d: Mass Transfer finished to %s, completed %s'%(self.unique_id, dps_location.show_ip(), fcomplete) dcslib.dps_cluster_write_log(DpsLogLevels.INFO, message) mass_transfer_complete = False DpsCollection.global_lock.acquire() while True: try: if self.mass_transfer is None: break #Don't make self.mass_transfer None till activate has been exchanged #self.mass_transfer = None DpsCollection.mass_transfer_lock.acquire() try: DpsCollection.MassTransfer_Active -= weight if DpsCollection.MassTransfer_Active < 0: message = 'Domain %s: Mass Transfer finished, accounting problem with mass transfer count %s' dcslib.dps_cluster_write_log(DpsLogLevels.CRITICAL, message) except Exception: pass DpsCollection.mass_transfer_lock.release() if fcomplete and self.valid: self.mass_transfer_forward_nodes[dps_location.ip_value] = dps_location mass_transfer_complete = True except Exception: pass break DpsCollection.global_lock.release() if mass_transfer_complete: message = 'Domain %s: Mass Transfer Complete, sending Activate message to remote Node %s'%(self.unique_id, dps_location.show_ip()) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) ret = dcslib.dps_domain_activate_on_node(dps_location.ip_value_packed, self.unique_id, self.replication_factor) if ret != 0: message = 'Domain %s: Activate on Node %s failed, clearing local forwarding cache'%(self.unique_id, dps_location.show_ip()) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) try: del self.mass_transfer_forward_nodes[dps_location.ip_value] except Exception: pass else: message = 'Domain %s: Mass Transfer canceled, sending Deactivate message to remote Node %s'%(self.unique_id, dps_location.show_ip()) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) ret = dcslib.dps_domain_deactivate_on_node(dps_location.ip_value_packed, self.unique_id) if ret != 0: message = 'Domain %s: Deactivate on Node %s Failed'%(self.unique_id, dps_location.show_ip()) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) DpsCollection.global_lock.acquire() self.mass_transfer = None DpsCollection.global_lock.release() return
def mass_transfer_finish(self, dps_location, fcomplete, weight): ''' This routine will be called by the Mass Transfer Callback when mass transfer of data to remote node is done. @param dps_location: The location of the remote node @type dps_location: IPAddressLocation @param fcomplete: Indicates if the transfer was complete or had to be cancelled due to some issue @type fcomplete: False @param weight: The weight of the mass transfer @type weight: Integer ''' message = 'Domain %d: Mass Transfer finished to %s, completed %s' % ( self.unique_id, dps_location.show_ip(), fcomplete) dcslib.dps_cluster_write_log(DpsLogLevels.INFO, message) mass_transfer_complete = False DpsCollection.global_lock.acquire() while True: try: if self.mass_transfer is None: break #Don't make self.mass_transfer None till activate has been exchanged #self.mass_transfer = None DpsCollection.mass_transfer_lock.acquire() try: DpsCollection.MassTransfer_Active -= weight if DpsCollection.MassTransfer_Active < 0: message = 'Domain %s: Mass Transfer finished, accounting problem with mass transfer count %s' dcslib.dps_cluster_write_log(DpsLogLevels.CRITICAL, message) except Exception: pass DpsCollection.mass_transfer_lock.release() if fcomplete and self.valid: self.mass_transfer_forward_nodes[ dps_location.ip_value] = dps_location mass_transfer_complete = True except Exception: pass break DpsCollection.global_lock.release() if mass_transfer_complete: message = 'Domain %s: Mass Transfer Complete, sending Activate message to remote Node %s' % ( self.unique_id, dps_location.show_ip()) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) ret = dcslib.dps_domain_activate_on_node( dps_location.ip_value_packed, self.unique_id, self.replication_factor) if ret != 0: message = 'Domain %s: Activate on Node %s failed, clearing local forwarding cache' % ( self.unique_id, dps_location.show_ip()) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) try: del self.mass_transfer_forward_nodes[dps_location.ip_value] except Exception: pass else: message = 'Domain %s: Mass Transfer canceled, sending Deactivate message to remote Node %s' % ( self.unique_id, dps_location.show_ip()) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) ret = dcslib.dps_domain_deactivate_on_node( dps_location.ip_value_packed, self.unique_id) if ret != 0: message = 'Domain %s: Deactivate on Node %s Failed' % ( self.unique_id, dps_location.show_ip()) dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message) DpsCollection.global_lock.acquire() self.mass_transfer = None DpsCollection.global_lock.release() return
def load_balance_domain(node_high, node_low): ''' This routine determines which domain can be moved from node1 to node2 or vice-versa that will minimize the load differential between the two i.e. minimize |load(node_high) - load(node_low)| @param node_high: The Higher Loaded Node @type node_high: DPSNodeStatistics @param node_low: The Lower Loaded Node @type node_low: DPSNodeStatistics @return: domain_id @rtype: Integer @raise: Exception if node_high is already lower than node_low or no domain is found ''' #print 'load_balance_domain: Enter High %s, Low %s\r'%(node_high.location.show_ip(), # node_low.location.show_ip()) load_high = DPSNodeDomainStatistics.load_array( node_high.domain_statistics.values()) load_low = DPSNodeDomainStatistics.load_array( node_low.domain_statistics.values()) #print 'load_balance_domain: load_high %s, load_low %s\r'%(load_high, load_low) if load_high <= load_low: raise Exception( 'Load on Node %s[%.2f], already lower than Node %s[%.2f]' % (node_high.location.show_ip(), load_high, node_low.location.show_ip(), load_low)) domain_id = None domain_ids = {} for node_domain in node_high.domain_statistics.values(): #Check if this domain is present in the lower node domain_id = node_domain.domain_id try: node_domain_low = node_low.domain_statistics[domain_id] continue except Exception: pass #Determine the Load if this domain moved from node_high to node_low load_domain = node_domain.load() #print 'Loop: domain %s, load %s\r'%(node_domain.domain_id, load_domain) if load_domain > load_high: message = 'Coding Error? Load on domain %s[%.2f] on node %s higher the load on node %.2f' % ( node_domain.domain_id, load_domain, node_high.location.show_ip(), load_high, ) dcslib.dps_cluster_write_log(DpsLogLevels.WARNING, message) continue load_high_new = load_high - load_domain load_low_new = load_low + load_domain load_diff_new = abs(load_high_new - load_low_new) domain_ids[node_domain] = load_diff_new if domain_id is None: raise Exception('No suitable domain found') #Sort the List based on values domain_tuples = sorted([(value, key) for (key, value) in domain_ids.items()]) #Only chose the 10 differential lowest domains i.e. the domains that cause max swing in load diff domain_tuples = domain_tuples[:10] print 'domain_tuples: %s\r' % domain_tuples domain_list = [] load_high_new = load_high load_low_new = load_low for domain_tuple in domain_tuples: node_domain = domain_tuple[1] domain_id = node_domain.domain_id load_domain = node_domain.load() load_high_new -= load_domain load_low_new += load_domain if load_low_new > DPSNodeStatValues.Heavy_Load: break if load_high_new < load_low_new: break domain_list.append(domain_id) #print 'load_balance_domain: Exit domain_id %s\r'%domain_id if len(domain_list) == 0: raise Exception('No suitable domain found') print 'domain_list: %s\r' % domain_list return domain_list