def SupplierSendFile(self, fileName, packetID, ownerID, callOnAck=None, callOnFail=None):
if self.shutdown:
if _Debug:
lg.out(_DebugLevel, "io_throttle.SupplierSendFile finishing to %s, shutdown is True" % self.remoteName)
return False
if contact_status.isOffline(self.remoteID):
if _Debug:
lg.out(_DebugLevel, "io_throttle.SupplierSendFile %s is offline, so packet %s is failed" % (
self.remoteName, packetID))
if callOnFail is not None:
reactor.callLater(0, callOnFail, self.remoteID, packetID, 'offline')
return False
if packetID in self.fileSendQueue:
lg.warn("packet %s already in the queue for %s" % (packetID, self.remoteName))
if callOnFail is not None:
reactor.callLater(0, callOnFail, self.remoteID, packetID, 'in queue')
return False
self.fileSendQueue.append(packetID)
self.fileSendDict[packetID] = FileToSend(
fileName,
packetID,
self.remoteID,
ownerID,
callOnAck,
callOnFail,)
if _Debug:
lg.out(_DebugLevel, "io_throttle.SupplierSendFile %s to %s, %d queued items" % (
packetID, self.remoteName, len(self.fileSendQueue)))
# reactor.callLater(0, self.DoSend)
self.DoSend()
return True
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
disconnected_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(6, 'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
elif sc.state == 'DISCONNECTED':
disconnected_suppliers.add(supplier_idurl)
if contact_status.isOffline(supplier_idurl):
disconnected_suppliers.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(
settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
if disconnected_suppliers:
from raid import eccmap
if len(disconnected_suppliers) >= eccmap.GetFireHireErrors(settings.getSuppliersNumberDesired()):
result.update(disconnected_suppliers)
result = list(result)
lg.out(6, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
to_be_fired = list(set(_SuppliersToFire))
_SuppliersToFire = []
if to_be_fired:
lg.warn('going to fire %d suppliers from external request' %
len(to_be_fired))
self.automat('made-decision', to_be_fired)
return
potentialy_fired = set()
connected_suppliers = set()
disconnected_suppliers = set()
requested_suppliers = set()
online_suppliers = set()
offline_suppliers = set()
redundant_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers() or None in contactsdb.suppliers():
lg.warn('SKIP, found empty supplier')
self.automat('made-decision', [])
return
number_desired = settings.getSuppliersNumberDesired()
for supplier_idurl in contactsdb.suppliers():
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
lg.warn('SKIP, supplier connector for supplier %s not exist' %
supplier_idurl)
continue
if sc.state == 'NO_SERVICE':
lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
disconnected_suppliers.add(supplier_idurl)
potentialy_fired.add(supplier_idurl)
elif sc.state == 'CONNECTED':
connected_suppliers.add(supplier_idurl)
elif sc.state in [
'DISCONNECTED',
'REFUSE',
]:
disconnected_suppliers.add(supplier_idurl)
# elif sc.state in ['QUEUE?', 'REQUEST', ]:
# requested_suppliers.add(supplier_idurl)
if contact_status.isOffline(supplier_idurl):
offline_suppliers.add(supplier_idurl)
elif contact_status.isOnline(supplier_idurl):
online_suppliers.add(supplier_idurl)
elif contact_status.isCheckingNow(supplier_idurl):
requested_suppliers.add(supplier_idurl)
if contactsdb.num_suppliers() > number_desired:
for supplier_index in range(number_desired,
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
lg.warn('found "REDUNDANT" supplier %s at position %d' % (
idurl,
supplier_index,
))
potentialy_fired.add(idurl)
redundant_suppliers.add(idurl)
else:
lg.warn('supplier at position %d not exist' %
supplier_index)
if not connected_suppliers or not online_suppliers:
lg.warn('SKIP, no ONLINE suppliers found at the moment')
self.automat('made-decision', [])
return
if requested_suppliers:
lg.warn('SKIP, still waiting response from some of suppliers')
self.automat('made-decision', [])
return
if redundant_suppliers:
result = list(redundant_suppliers)
lg.info('will replace redundant suppliers: %s' % result)
self.automat('made-decision', result)
return
if not disconnected_suppliers:
lg.warn('SKIP, no OFFLINE suppliers found at the moment')
# TODO: add more conditions to fire "slow" suppliers
self.automat('made-decision', [])
return
if len(offline_suppliers) + len(online_suppliers) != number_desired:
lg.warn('SKIP, offline + online != total count: %s %s %s' %
(offline_suppliers, online_suppliers, number_desired))
self.automat('made-decision', [])
return
from raid import eccmap
max_offline_suppliers_count = eccmap.GetCorrectableErrors(
number_desired)
if len(offline_suppliers) > max_offline_suppliers_count:
lg.warn(
'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
len(offline_suppliers),
max_offline_suppliers_count,
))
self.automat('made-decision', [])
return
critical_offline_suppliers_count = eccmap.GetFireHireErrors(
number_desired)
# TODO: temporary disabled because of an issue: too aggressive replacing suppliers who still have the data
if False: # len(offline_suppliers) >= critical_offline_suppliers_count:
one_dead_supplier = offline_suppliers.pop()
lg.warn(
'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
% (
one_dead_supplier,
critical_offline_suppliers_count,
))
potentialy_fired.add(one_dead_supplier)
if not potentialy_fired:
lg.out(
6,
'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!'
)
self.automat('made-decision', [])
return
# only replace suppliers one by one at the moment
result = list(potentialy_fired)
lg.info('will replace supplier %s' % result[0])
self.automat('made-decision', [
result[0],
])
def doRequestPackets(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.backup_id,
self.block_number,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if packetID not in self.AlreadyRequestedCounts:
self.AlreadyRequestedCounts[packetID] = 0
self.AlreadyRequestedCounts[packetID] += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.creator_id, packetID,
self.creator_id, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.block_number))
else:
if not already_requested:
lg.warn('no requests made for block %d' % self.block_number)
self.automat('request-failed', None)
else:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.block_number))
if self.AlreadyRequestedCounts:
all_counts = sorted(self.AlreadyRequestedCounts.values())
if all_counts[0] > 100:
lg.warn('too much requests made for block %d' %
self.block_number)
self.automat('request-failed', None)
def doRequestPackets(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.BackupID,
self.BlockNumber,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
lg.warn('bad supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
lg.warn('bad supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.CreatorID, packetID,
self.CreatorID, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.BlockNumber))
else:
if already_requested:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.BlockNumber))
else:
lg.warn('no requests made for block %d' % self.BlockNumber)
self.automat('request-failed', None)