def heavy_load(self):
'''
This routine returns if this node is heavily loaded.
@return: True if heavy load, False otherwise
@rtype: Boolean
'''
my_load = DPSNodeDomainStatistics.load_array(self.domain_statistics.values())
message = 'Node %s, Load %s'%(self.location.show_ip(), my_load)
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
if my_load > DPSNodeStatValues.Heavy_Load:
return (True, my_load)
else:
return (False, my_load)
def heavy_load(self):
'''
This routine returns if this node is heavily loaded.
@return: True if heavy load, False otherwise
@rtype: Boolean
'''
my_load = DPSNodeDomainStatistics.load_array(
self.domain_statistics.values())
message = 'Node %s, Load %s' % (self.location.show_ip(), my_load)
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
if my_load > DPSNodeStatValues.Heavy_Load:
return (True, my_load)
else:
return (False, my_load)
def load_balance_domain(node_high, node_low):
'''
This routine determines which domain can be moved from node1 to
node2 or vice-versa that will minimize the load differential
between the two i.e. minimize |load(node_high) - load(node_low)|
@param node_high: The Higher Loaded Node
@type node_high: DPSNodeStatistics
@param node_low: The Lower Loaded Node
@type node_low: DPSNodeStatistics
@return: domain_id
@rtype: Integer
@raise: Exception if node_high is already lower than node_low
or no domain is found
'''
#print 'load_balance_domain: Enter High %s, Low %s\r'%(node_high.location.show_ip(),
# node_low.location.show_ip())
load_high = DPSNodeDomainStatistics.load_array(node_high.domain_statistics.values())
load_low = DPSNodeDomainStatistics.load_array(node_low.domain_statistics.values())
#print 'load_balance_domain: load_high %s, load_low %s\r'%(load_high, load_low)
if load_high <= load_low:
raise Exception('Load on Node %s[%.2f], already lower than Node %s[%.2f]'%(node_high.location.show_ip(),
load_high,
node_low.location.show_ip(),
load_low))
domain_id = None
domain_ids = {}
for node_domain in node_high.domain_statistics.values():
#Check if this domain is present in the lower node
domain_id = node_domain.domain_id
try:
node_domain_low = node_low.domain_statistics[domain_id]
continue
except Exception:
pass
#Determine the Load if this domain moved from node_high to node_low
load_domain = node_domain.load()
#print 'Loop: domain %s, load %s\r'%(node_domain.domain_id, load_domain)
if load_domain > load_high:
message = 'Coding Error? Load on domain %s[%.2f] on node %s higher the load on node %.2f'%(node_domain.domain_id,
load_domain,
node_high.location.show_ip(),
load_high,
)
dcslib.dps_cluster_write_log(DpsLogLevels.WARNING, message)
continue
load_high_new = load_high - load_domain
load_low_new = load_low + load_domain
load_diff_new = abs(load_high_new - load_low_new)
domain_ids[node_domain] = load_diff_new
if domain_id is None:
raise Exception('No suitable domain found')
#Sort the List based on values
domain_tuples = sorted([(value,key) for (key,value) in domain_ids.items()])
#Only chose the 10 differential lowest domains i.e. the domains that cause max swing in load diff
domain_tuples = domain_tuples[:10]
print 'domain_tuples: %s\r'%domain_tuples
domain_list = []
load_high_new = load_high
load_low_new = load_low
for domain_tuple in domain_tuples:
node_domain = domain_tuple[1]
domain_id = node_domain.domain_id
load_domain = node_domain.load()
load_high_new -= load_domain
load_low_new += load_domain
if load_low_new > DPSNodeStatValues.Heavy_Load:
break
if load_high_new < load_low_new:
break
domain_list.append(domain_id)
#print 'load_balance_domain: Exit domain_id %s\r'%domain_id
if len(domain_list) == 0:
raise Exception('No suitable domain found')
print 'domain_list: %s\r'%domain_list
return domain_list
def mass_transfer_finish(self, dps_location, fcomplete, weight):
'''
This routine will be called by the Mass Transfer Callback when mass transfer
of data to remote node is done.
@param dps_location: The location of the remote node
@type dps_location: IPAddressLocation
@param fcomplete: Indicates if the transfer was complete or had to be cancelled
due to some issue
@type fcomplete: False
@param weight: The weight of the mass transfer
@type weight: Integer
'''
message = 'Domain %d: Mass Transfer finished to %s, completed %s'%(self.unique_id,
dps_location.show_ip(),
fcomplete)
dcslib.dps_cluster_write_log(DpsLogLevels.INFO, message)
mass_transfer_complete = False
DpsCollection.global_lock.acquire()
while True:
try:
if self.mass_transfer is None:
break
#Don't make self.mass_transfer None till activate has been exchanged
#self.mass_transfer = None
DpsCollection.mass_transfer_lock.acquire()
try:
DpsCollection.MassTransfer_Active -= weight
if DpsCollection.MassTransfer_Active < 0:
message = 'Domain %s: Mass Transfer finished, accounting problem with mass transfer count %s'
dcslib.dps_cluster_write_log(DpsLogLevels.CRITICAL, message)
except Exception:
pass
DpsCollection.mass_transfer_lock.release()
if fcomplete and self.valid:
self.mass_transfer_forward_nodes[dps_location.ip_value] = dps_location
mass_transfer_complete = True
except Exception:
pass
break
DpsCollection.global_lock.release()
if mass_transfer_complete:
message = 'Domain %s: Mass Transfer Complete, sending Activate message to remote Node %s'%(self.unique_id,
dps_location.show_ip())
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
ret = dcslib.dps_domain_activate_on_node(dps_location.ip_value_packed,
self.unique_id,
self.replication_factor)
if ret != 0:
message = 'Domain %s: Activate on Node %s failed, clearing local forwarding cache'%(self.unique_id,
dps_location.show_ip())
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
try:
del self.mass_transfer_forward_nodes[dps_location.ip_value]
except Exception:
pass
else:
message = 'Domain %s: Mass Transfer canceled, sending Deactivate message to remote Node %s'%(self.unique_id,
dps_location.show_ip())
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
ret = dcslib.dps_domain_deactivate_on_node(dps_location.ip_value_packed,
self.unique_id)
if ret != 0:
message = 'Domain %s: Deactivate on Node %s Failed'%(self.unique_id,
dps_location.show_ip())
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
DpsCollection.global_lock.acquire()
self.mass_transfer = None
DpsCollection.global_lock.release()
return
def mass_transfer_finish(self, dps_location, fcomplete, weight):
'''
This routine will be called by the Mass Transfer Callback when mass transfer
of data to remote node is done.
@param dps_location: The location of the remote node
@type dps_location: IPAddressLocation
@param fcomplete: Indicates if the transfer was complete or had to be cancelled
due to some issue
@type fcomplete: False
@param weight: The weight of the mass transfer
@type weight: Integer
'''
message = 'Domain %d: Mass Transfer finished to %s, completed %s' % (
self.unique_id, dps_location.show_ip(), fcomplete)
dcslib.dps_cluster_write_log(DpsLogLevels.INFO, message)
mass_transfer_complete = False
DpsCollection.global_lock.acquire()
while True:
try:
if self.mass_transfer is None:
break
#Don't make self.mass_transfer None till activate has been exchanged
#self.mass_transfer = None
DpsCollection.mass_transfer_lock.acquire()
try:
DpsCollection.MassTransfer_Active -= weight
if DpsCollection.MassTransfer_Active < 0:
message = 'Domain %s: Mass Transfer finished, accounting problem with mass transfer count %s'
dcslib.dps_cluster_write_log(DpsLogLevels.CRITICAL,
message)
except Exception:
pass
DpsCollection.mass_transfer_lock.release()
if fcomplete and self.valid:
self.mass_transfer_forward_nodes[
dps_location.ip_value] = dps_location
mass_transfer_complete = True
except Exception:
pass
break
DpsCollection.global_lock.release()
if mass_transfer_complete:
message = 'Domain %s: Mass Transfer Complete, sending Activate message to remote Node %s' % (
self.unique_id, dps_location.show_ip())
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
ret = dcslib.dps_domain_activate_on_node(
dps_location.ip_value_packed, self.unique_id,
self.replication_factor)
if ret != 0:
message = 'Domain %s: Activate on Node %s failed, clearing local forwarding cache' % (
self.unique_id, dps_location.show_ip())
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
try:
del self.mass_transfer_forward_nodes[dps_location.ip_value]
except Exception:
pass
else:
message = 'Domain %s: Mass Transfer canceled, sending Deactivate message to remote Node %s' % (
self.unique_id, dps_location.show_ip())
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
ret = dcslib.dps_domain_deactivate_on_node(
dps_location.ip_value_packed, self.unique_id)
if ret != 0:
message = 'Domain %s: Deactivate on Node %s Failed' % (
self.unique_id, dps_location.show_ip())
dcslib.dps_cluster_write_log(DpsLogLevels.NOTICE, message)
DpsCollection.global_lock.acquire()
self.mass_transfer = None
DpsCollection.global_lock.release()
return
def load_balance_domain(node_high, node_low):
'''
This routine determines which domain can be moved from node1 to
node2 or vice-versa that will minimize the load differential
between the two i.e. minimize |load(node_high) - load(node_low)|
@param node_high: The Higher Loaded Node
@type node_high: DPSNodeStatistics
@param node_low: The Lower Loaded Node
@type node_low: DPSNodeStatistics
@return: domain_id
@rtype: Integer
@raise: Exception if node_high is already lower than node_low
or no domain is found
'''
#print 'load_balance_domain: Enter High %s, Low %s\r'%(node_high.location.show_ip(),
# node_low.location.show_ip())
load_high = DPSNodeDomainStatistics.load_array(
node_high.domain_statistics.values())
load_low = DPSNodeDomainStatistics.load_array(
node_low.domain_statistics.values())
#print 'load_balance_domain: load_high %s, load_low %s\r'%(load_high, load_low)
if load_high <= load_low:
raise Exception(
'Load on Node %s[%.2f], already lower than Node %s[%.2f]' %
(node_high.location.show_ip(), load_high,
node_low.location.show_ip(), load_low))
domain_id = None
domain_ids = {}
for node_domain in node_high.domain_statistics.values():
#Check if this domain is present in the lower node
domain_id = node_domain.domain_id
try:
node_domain_low = node_low.domain_statistics[domain_id]
continue
except Exception:
pass
#Determine the Load if this domain moved from node_high to node_low
load_domain = node_domain.load()
#print 'Loop: domain %s, load %s\r'%(node_domain.domain_id, load_domain)
if load_domain > load_high:
message = 'Coding Error? Load on domain %s[%.2f] on node %s higher the load on node %.2f' % (
node_domain.domain_id,
load_domain,
node_high.location.show_ip(),
load_high,
)
dcslib.dps_cluster_write_log(DpsLogLevels.WARNING, message)
continue
load_high_new = load_high - load_domain
load_low_new = load_low + load_domain
load_diff_new = abs(load_high_new - load_low_new)
domain_ids[node_domain] = load_diff_new
if domain_id is None:
raise Exception('No suitable domain found')
#Sort the List based on values
domain_tuples = sorted([(value, key)
for (key, value) in domain_ids.items()])
#Only chose the 10 differential lowest domains i.e. the domains that cause max swing in load diff
domain_tuples = domain_tuples[:10]
print 'domain_tuples: %s\r' % domain_tuples
domain_list = []
load_high_new = load_high
load_low_new = load_low
for domain_tuple in domain_tuples:
node_domain = domain_tuple[1]
domain_id = node_domain.domain_id
load_domain = node_domain.load()
load_high_new -= load_domain
load_low_new += load_domain
if load_low_new > DPSNodeStatValues.Heavy_Load:
break
if load_high_new < load_low_new:
break
domain_list.append(domain_id)
#print 'load_balance_domain: Exit domain_id %s\r'%domain_id
if len(domain_list) == 0:
raise Exception('No suitable domain found')
print 'domain_list: %s\r' % domain_list
return domain_list