def doRequestPackets(self, arg):
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
continue
if not self.OnHandData[SupplierNumber] and contact_status.isOnline(
SupplierID):
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
continue
if not self.OnHandParity[
SupplierNumber] and contact_status.isOnline(SupplierID):
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')))
for SupplierID, packetID in packetsToRequest:
io_throttle.QueueRequestFile(self._on_inbox_packet, self.CreatorID,
packetID, self.CreatorID, SupplierID)
lg.out(
6, "restore.doRequestPackets requested %d packets for block %d" %
(len(packetsToRequest), self.BlockNumber))
del packetsToRequest
self.automat('request-done')
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(10,
'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
result = list(result)
lg.out(10, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def doFindNewSupplier(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'fire_hire.doFindNewSupplier')
position_for_new_supplier = None
for pos in range(settings.getSuppliersNumberDesired()):
if pos in self.hire_list:
continue
supplier_idurl = contactsdb.supplier(pos)
if not supplier_idurl:
lg.info('found empty supplier at position %d and going to find new supplier on that position' % pos)
position_for_new_supplier = pos
break
if supplier_idurl in self.dismiss_list:
lg.info('going to find new supplier on existing position %d to replace supplier %s' % (
pos, supplier_idurl, ))
position_for_new_supplier = pos
break
if position_for_new_supplier is None:
lg.err('did not found position for new supplier')
self.automat('search-failed')
return
self.hire_list.append(position_for_new_supplier)
supplier_finder.A(
'start',
family_position=position_for_new_supplier,
ecc_map=eccmap.Current().name,
family_snapshot=contactsdb.suppliers(),
)
def SendListFiles(target_supplier, customer_idurl=None, key_id=None, wide=False, callbacks={}):
"""
This is used as a request method from your supplier : if you send him a ListFiles() packet
he will reply you with a list of stored files in a Files() packet.
"""
MyID = my_id.getLocalID()
if not customer_idurl:
customer_idurl = MyID
if not str(target_supplier).isdigit():
RemoteID = target_supplier
else:
RemoteID = contactsdb.supplier(target_supplier, customer_idurl=customer_idurl)
if not RemoteID:
lg.warn("RemoteID is empty target_supplier=%s" % str(target_supplier))
return None
if _Debug:
lg.out(_DebugLevel, "p2p_service.SendListFiles to %s" % nameurl.GetName(RemoteID))
if not key_id:
key_id = global_id.MakeGlobalID(idurl=customer_idurl, key_alias='customer')
PacketID = "%s:%s" % (key_id, packetid.UniqueID(), )
Payload = settings.ListFilesFormat()
result = signed.Packet(
Command=commands.ListFiles(),
OwnerID=MyID,
CreatorID=MyID,
PacketID=PacketID,
Payload=Payload,
RemoteID=RemoteID,
)
gateway.outbox(result, wide=wide, callbacks=callbacks)
return result
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(
settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
result = list(result)
lg.out(10, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def _send(index, payload, delay):
global _SlowSendIsWorking
idurl = contactsdb.supplier(index, customer_idurl=customer_idurl)
if not idurl:
_SlowSendIsWorking = False
return
# transport_control.ClearAliveTime(idurl)
SendToID(idurl, Payload=payload, wide=True)
reactor.callLater(delay, _send, index + 1, payload, delay)
def doFindNewSupplier(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r'
%
(settings.getSuppliersNumberDesired(), contactsdb.suppliers()))
from p2p import network_connector
if network_connector.A().state != 'CONNECTED':
if _Debug:
lg.out(
_DebugLevel,
' network_connector is not CONNECTED at the moment, SKIP'
)
self.automat('search-failed')
return
position_for_new_supplier = None
for pos in range(settings.getSuppliersNumberDesired()):
if pos in self.hire_list:
continue
supplier_idurl = contactsdb.supplier(pos)
if not supplier_idurl:
lg.info(
'found empty supplier at position %d and going to find new supplier on that position'
% pos)
position_for_new_supplier = pos
break
if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False):
lg.info(
'going to find new supplier on existing position %d to replace supplier %s'
% (
pos,
supplier_idurl,
))
position_for_new_supplier = pos
break
if position_for_new_supplier is None:
lg.err('did not found position for new supplier')
self.automat('search-failed')
return
from customer import supplier_finder
for idurl_txt in strng.to_text(config.conf().getData(
'services/employer/candidates')).split(','):
if idurl_txt.strip():
supplier_finder.AddSupplierToHire(idurl_txt)
self.hire_list.append(position_for_new_supplier)
supplier_finder.A(
'start',
family_position=position_for_new_supplier,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(contactsdb.suppliers()),
)
def isRequestQueueEmpty(self, arg):
"""
Condition method.
"""
from customer import io_throttle
# supplierSet = backup_matrix.suppliers_set()
for supplierNum in range(contactsdb.num_suppliers()):
supplierID = contactsdb.supplier(supplierNum)
if io_throttle.HasBackupIDInRequestQueue(
supplierID, self.currentBackupID):
return False
return True
def doRemoveTempFile(self, *args, **kwargs):
"""
Action method.
"""
if not args or not len(args) > 1:
return
filename = args[1]
if filename:
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
from storage import backup_matrix
from storage import backup_rebuilder
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.backup_id:
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in range(
contactsdb.num_suppliers(customer_idurl=self.customer_idurl)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.customer_idurl)
if not supplierIDURL:
continue
for dataORparity in [
'Data',
'Parity',
]:
packetID = packetid.MakePacketID(self.backup_id,
self.block_number,
supplierNum, dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.backup_id, self.block_number,
*args, **kwargs)
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRemoveTempFile %d files were removed' %
count)
def SuppliersChangedNumbers(oldSupplierList):
"""
Return list of positions of changed suppliers, say if suppliers 1 and 3
were changed it should return [1,3].
"""
changedList = []
for i in xrange(len(oldSupplierList)):
suplier_idurl = oldSupplierList[i]
if not suplier_idurl:
continue
if contactsdb.supplier(i) != suplier_idurl:
changedList.append(i)
return changedList
def ScanBlocksToRemove(backupID, check_all_suppliers=True):
"""
This method compare both matrixes and found pieces which is present on both
sides.
If remote supplier got that file it can be removed from the local
HDD.
"""
from customer import io_throttle
lg.out(10, "backup_matrix.ScanBlocksToRemove for %s" % backupID)
packets = []
localMaxBlockNum = local_max_block_numbers().get(backupID, -1)
if backupID not in remote_files() or backupID not in local_files():
# no info about this backup yet - skip
return packets
for blockNum in xrange(localMaxBlockNum + 1):
localArray = {"Data": GetLocalDataArray(backupID, blockNum), "Parity": GetLocalParityArray(backupID, blockNum)}
remoteArray = {
"Data": GetRemoteDataArray(backupID, blockNum),
"Parity": GetRemoteParityArray(backupID, blockNum),
}
if (0 in remoteArray["Data"]) or (0 in remoteArray["Parity"]):
# if some supplier do not have some data for that block - do not remove any local files for that block!
# we do remove the local files only when we sure all suppliers got the all data pieces
continue
if (-1 in remoteArray["Data"]) or (-1 in remoteArray["Parity"]):
# also if we do not have any info about this block for some supplier do not remove other local pieces
continue
for supplierNum in xrange(contactsdb.num_suppliers()):
supplierIDURL = contactsdb.supplier(supplierNum)
if not supplierIDURL:
# supplier is unknown - skip
continue
for dataORparity in ["Data", "Parity"]:
packetID = packetid.MakePacketID(backupID, blockNum, supplierNum, dataORparity)
if io_throttle.HasPacketInSendQueue(supplierIDURL, packetID):
# if we do sending the packet at the moment - skip
continue
if localArray[dataORparity][supplierNum] == 1:
packets.append(packetID)
# lg.out(10, ' mark to remove %s, blockNum:%d remote:%s local:%s' % (packetID, blockNum, str(remoteArray), str(localArray)))
# if check_all_suppliers:
# if localArray[dataORparity][supplierNum] == 1:
# packets.append(packetID)
# else:
# if remoteArray[dataORparity][supplierNum] == 1 and localArray[dataORparity][supplierNum] == 1:
# packets.append(packetID)
return packets
def SendRequestListFiles(supplierNumORidurl):
if not str(supplierNumORidurl).isdigit():
RemoteID = supplierNumORidurl
else:
RemoteID = contactsdb.supplier(supplierNumORidurl)
if not RemoteID:
lg.warn("RemoteID is empty supplierNumORidurl=%s" % str(supplierNumORidurl))
return None
if _Debug:
lg.out(_DebugLevel, "p2p_service.SendRequestListFiles [%s]" % nameurl.GetName(RemoteID))
MyID = my_id.getLocalID()
PacketID = packetid.UniqueID()
Payload = settings.ListFilesFormat()
result = signed.Packet(commands.ListFiles(), MyID, MyID, PacketID, Payload, RemoteID)
gateway.outbox(result)
return result
def SendListFiles(target_supplier, customer_idurl=None, key_id=None, query_items=[], wide=False, callbacks={}, timeout=None):
"""
This is used as a request method from your supplier : if you send him a ListFiles() packet
he will reply you with a list of stored files in a Files() packet.
"""
MyID = my_id.getLocalID()
if not customer_idurl:
customer_idurl = MyID
if not str(target_supplier).isdigit():
RemoteID = target_supplier
else:
RemoteID = contactsdb.supplier(target_supplier, customer_idurl=customer_idurl)
if not RemoteID:
lg.warn("RemoteID is empty target_supplier=%s" % str(target_supplier))
return None
if not key_id:
# key_id = global_id.MakeGlobalID(idurl=customer_idurl, key_alias='customer')
# TODO: due to issue with "customer" key backup/restore decided to always use my "master" key
# to retrieve my list files info from supplier
# expect remote user always poses my master public key from my identity.
# probably require more work to build more reliable solution without using my master key at all
# when my identity rotated supplier first needs to receive my new identity and then sending ListFiles()
key_id = my_id.getGlobalID(key_alias='master')
if not my_keys.is_key_registered(key_id) or not my_keys.is_key_private(key_id):
lg.warn('key %r not exist or public, my "master" key to be used with ListFiles() packet' % key_id)
key_id = my_id.getGlobalID(key_alias='master')
PacketID = "%s:%s" % (key_id, packetid.UniqueID(), )
if not query_items:
query_items = ['*', ]
Payload = serialization.DictToBytes({'items': query_items, })
if _Debug:
lg.out(_DebugLevel, "p2p_service.SendListFiles %r to %s with query : %r" % (
PacketID, nameurl.GetName(RemoteID), query_items, ))
result = signed.Packet(
Command=commands.ListFiles(),
OwnerID=MyID,
CreatorID=MyID,
PacketID=PacketID,
Payload=Payload,
RemoteID=RemoteID,
)
gateway.outbox(result, wide=wide, callbacks=callbacks, response_timeout=timeout)
return result
def GetActiveArray():
"""
Loops all suppliers and returns who is alive at the moment.
Return a list with integers: 0 for offline suppler and 1 if he is
available right now. Uses ``p2p.contact_status.isOnline()`` to see
the current state of supplier.
"""
from p2p import contact_status
activeArray = [0] * contactsdb.num_suppliers()
for i in xrange(contactsdb.num_suppliers()):
suplier_idurl = contactsdb.supplier(i)
if not suplier_idurl:
continue
if contact_status.isOnline(suplier_idurl):
activeArray[i] = 1
else:
activeArray[i] = 0
return activeArray
def doRemoveTempFile(self, arg):
try:
filename = arg[1]
except:
return
tmpfile.throw_out(filename, 'block restored')
if settings.getBackupsKeepLocalCopies():
return
import backup_rebuilder
import backup_matrix
if not backup_rebuilder.ReadStoppedFlag():
if backup_rebuilder.A().currentBackupID is not None:
if backup_rebuilder.A().currentBackupID == self.BackupID:
lg.out(
6,
'restore.doRemoveTempFile SKIP because rebuilding in process'
)
return
count = 0
for supplierNum in xrange(
contactsdb.num_suppliers(customer_idurl=self.CustomerIDURL)):
supplierIDURL = contactsdb.supplier(
supplierNum, customer_idurl=self.CustomerIDURL)
if not supplierIDURL:
continue
for dataORparity in ['Data', 'Parity']:
packetID = packetid.MakePacketID(self.BackupID,
self.BlockNumber, supplierNum,
dataORparity)
customer, remotePath = packetid.SplitPacketID(packetID)
filename = os.path.join(settings.getLocalBackupsDir(),
customer, remotePath)
if os.path.isfile(filename):
try:
os.remove(filename)
except:
lg.exc()
continue
count += 1
backup_matrix.LocalBlockReport(self.BackupID, self.BlockNumber, arg)
lg.out(6, 'restore.doRemoveTempFile %d files were removed' % count)
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
disconnected_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(6, 'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
elif sc.state == 'DISCONNECTED':
disconnected_suppliers.add(supplier_idurl)
if contact_status.isOffline(supplier_idurl):
disconnected_suppliers.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(
settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
if disconnected_suppliers:
from raid import eccmap
if len(disconnected_suppliers) >= eccmap.GetFireHireErrors(settings.getSuppliersNumberDesired()):
result.update(disconnected_suppliers)
result = list(result)
lg.out(6, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def SendRequestListFiles(supplierNumORidurl, customer_idurl=None):
MyID = my_id.getLocalID()
if not customer_idurl:
customer_idurl = MyID
if not str(supplierNumORidurl).isdigit():
RemoteID = supplierNumORidurl
else:
RemoteID = contactsdb.supplier(supplierNumORidurl,
customer_idurl=customer_idurl)
if not RemoteID:
lg.warn("RemoteID is empty supplierNumORidurl=%s" %
str(supplierNumORidurl))
return None
if _Debug:
lg.out(
_DebugLevel, "p2p_service.SendRequestListFiles [%s]" %
nameurl.GetName(RemoteID))
PacketID = "%s:%s" % (global_id.UrlToGlobalID(customer_idurl),
packetid.UniqueID())
Payload = settings.ListFilesFormat()
result = signed.Packet(commands.ListFiles(), MyID, MyID, PacketID, Payload,
RemoteID)
gateway.outbox(result)
return result
def doRequestPackets(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.BackupID,
self.BlockNumber,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
lg.warn('bad supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
lg.warn('bad supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.CreatorID, packetID,
self.CreatorID, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.BlockNumber))
else:
if already_requested:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.BlockNumber))
else:
lg.warn('no requests made for block %d' % self.BlockNumber)
self.automat('request-failed', None)
def doRequestPackets(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.backup_id,
self.block_number,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if contact_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if packetID not in self.AlreadyRequestedCounts:
self.AlreadyRequestedCounts[packetID] = 0
self.AlreadyRequestedCounts[packetID] += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.creator_id, packetID,
self.creator_id, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.block_number))
else:
if not already_requested:
lg.warn('no requests made for block %d' % self.block_number)
self.automat('request-failed', None)
else:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.block_number))
if self.AlreadyRequestedCounts:
all_counts = sorted(self.AlreadyRequestedCounts.values())
if all_counts[0] > 100:
lg.warn('too much requests made for block %d' %
self.block_number)
self.automat('request-failed', None)
def _do_check_run_requests(self):
if _Debug:
lg.out(
_DebugLevel,
'restore_worker._do_check_run_requests for %s at block %d' % (
self.backup_id,
self.block_number,
))
from stream import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
request_packet_id = packetid.MakePacketID(self.backup_id,
self.block_number,
SupplierNumber, 'Data')
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, OnHandData is True for supplier %d' %
SupplierNumber)
if request_packet_id not in self.block_requests:
self.block_requests[request_packet_id] = True
continue
if request_packet_id in self.block_requests:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, request for packet %r already sent to IO queue for supplier %d'
% (
request_packet_id,
SupplierNumber,
))
continue
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if online_status.isOffline(SupplierID):
if _Debug:
lg.out(_DebugLevel,
' SKIP, offline supplier: %s' % SupplierID)
continue
packetsToRequest.append((
SupplierID,
request_packet_id,
))
for SupplierNumber in range(self.EccMap.paritysegments):
request_packet_id = packetid.MakePacketID(self.backup_id,
self.block_number,
SupplierNumber, 'Parity')
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, OnHandParity is True for supplier %d' %
SupplierNumber)
if request_packet_id not in self.block_requests:
self.block_requests[request_packet_id] = True
continue
if request_packet_id in self.block_requests:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, request for packet %r already sent to IO queue for supplier %d'
% (
request_packet_id,
SupplierNumber,
))
continue
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if online_status.isOffline(SupplierID):
if _Debug:
lg.out(_DebugLevel,
' SKIP, offline supplier: %s' % SupplierID)
continue
packetsToRequest.append((
SupplierID,
request_packet_id,
))
requests_made = 0
# already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
# already_requested += 1
# if packetID not in self.AlreadyRequestedCounts:
# self.AlreadyRequestedCounts[packetID] = 0
# self.AlreadyRequestedCounts[packetID] += 1
lg.warn('packet already in IO queue for supplier %s : %s' % (
SupplierID,
packetID,
))
continue
self.block_requests[packetID] = None
if io_throttle.QueueRequestFile(
callOnReceived=self._on_packet_request_result,
creatorID=self.creator_id,
packetID=packetID,
ownerID=self.creator_id, # self.customer_idurl,
remoteID=SupplierID,
):
requests_made += 1
else:
self.block_requests[packetID] = False
if _Debug:
lg.dbg(
_DebugLevel, 'sent request %r to %r, other requests: %r' %
(packetID, SupplierID, list(self.block_requests.values())))
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel,
" requested %d packets for block %d" % (
requests_made,
self.block_number,
))
return
current_block_requests_results = list(self.block_requests.values())
if _Debug:
lg.args(_DebugLevel,
current_results=current_block_requests_results)
pending_count = current_block_requests_results.count(None)
if pending_count > 0:
if _Debug:
lg.out(
_DebugLevel,
" nothing for request, currently %d pending packets for block %d"
% (
pending_count,
self.block_number,
))
return
failed_count = current_block_requests_results.count(False)
if failed_count > self.max_errors:
lg.err(
'all requests finished and %d packets failed, not possible to read data for block %d'
% (
failed_count,
self.block_number,
))
reactor.callLater(0, self.automat, 'request-failed',
None) # @UndefinedVariable
return
if _Debug:
lg.out(
_DebugLevel,
" all requests finished for block %d : %r" % (
self.block_number,
current_block_requests_results,
))
reactor.callLater(0, self.automat, 'request-finished',
None) # @UndefinedVariable
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
to_be_fired = list(set(_SuppliersToFire))
_SuppliersToFire = []
if to_be_fired:
lg.warn('going to fire %d suppliers from external request' %
len(to_be_fired))
self.automat('made-decision', to_be_fired)
return
potentialy_fired = set()
connected_suppliers = set()
disconnected_suppliers = set()
requested_suppliers = set()
online_suppliers = set()
offline_suppliers = set()
redundant_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers() or None in contactsdb.suppliers():
lg.warn('SKIP, found empty supplier')
self.automat('made-decision', [])
return
number_desired = settings.getSuppliersNumberDesired()
for supplier_idurl in contactsdb.suppliers():
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
lg.warn('SKIP, supplier connector for supplier %s not exist' %
supplier_idurl)
continue
if sc.state == 'NO_SERVICE':
lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
disconnected_suppliers.add(supplier_idurl)
potentialy_fired.add(supplier_idurl)
elif sc.state == 'CONNECTED':
connected_suppliers.add(supplier_idurl)
elif sc.state in [
'DISCONNECTED',
'REFUSE',
]:
disconnected_suppliers.add(supplier_idurl)
# elif sc.state in ['QUEUE?', 'REQUEST', ]:
# requested_suppliers.add(supplier_idurl)
if contact_status.isOffline(supplier_idurl):
offline_suppliers.add(supplier_idurl)
elif contact_status.isOnline(supplier_idurl):
online_suppliers.add(supplier_idurl)
elif contact_status.isCheckingNow(supplier_idurl):
requested_suppliers.add(supplier_idurl)
if contactsdb.num_suppliers() > number_desired:
for supplier_index in range(number_desired,
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
lg.warn('found "REDUNDANT" supplier %s at position %d' % (
idurl,
supplier_index,
))
potentialy_fired.add(idurl)
redundant_suppliers.add(idurl)
else:
lg.warn('supplier at position %d not exist' %
supplier_index)
if not connected_suppliers or not online_suppliers:
lg.warn('SKIP, no ONLINE suppliers found at the moment')
self.automat('made-decision', [])
return
if requested_suppliers:
lg.warn('SKIP, still waiting response from some of suppliers')
self.automat('made-decision', [])
return
if redundant_suppliers:
result = list(redundant_suppliers)
lg.info('will replace redundant suppliers: %s' % result)
self.automat('made-decision', result)
return
if not disconnected_suppliers:
lg.warn('SKIP, no OFFLINE suppliers found at the moment')
# TODO: add more conditions to fire "slow" suppliers
self.automat('made-decision', [])
return
if len(offline_suppliers) + len(online_suppliers) != number_desired:
lg.warn('SKIP, offline + online != total count: %s %s %s' %
(offline_suppliers, online_suppliers, number_desired))
self.automat('made-decision', [])
return
from raid import eccmap
max_offline_suppliers_count = eccmap.GetCorrectableErrors(
number_desired)
if len(offline_suppliers) > max_offline_suppliers_count:
lg.warn(
'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
len(offline_suppliers),
max_offline_suppliers_count,
))
self.automat('made-decision', [])
return
critical_offline_suppliers_count = eccmap.GetFireHireErrors(
number_desired)
# TODO: temporary disabled because of an issue: too aggressive replacing suppliers who still have the data
if False: # len(offline_suppliers) >= critical_offline_suppliers_count:
one_dead_supplier = offline_suppliers.pop()
lg.warn(
'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
% (
one_dead_supplier,
critical_offline_suppliers_count,
))
potentialy_fired.add(one_dead_supplier)
if not potentialy_fired:
lg.out(
6,
'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!'
)
self.automat('made-decision', [])
return
# only replace suppliers one by one at the moment
result = list(potentialy_fired)
lg.info('will replace supplier %s' % result[0])
self.automat('made-decision', [
result[0],
])
def _request_files(self):
from storage import backup_matrix
from customer import io_throttle
from customer import data_sender
self.missingPackets = 0
# here we want to request some packets before we start working to
# rebuild the missed blocks
availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL)
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL):
lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier')
self.automat('no-requests')
return
for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)):
supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL)
if not supplierID:
continue
requests_count = 0
# we do requests in reverse order because we start rebuilding from
# the last block
for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(
self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(
self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(
self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(
self.currentBackupID, blockNum)
if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity):
break
if supplierNum >= len(localData) or supplierNum >= len(localParity):
break
# if remote Data exist and is available because supplier is on-line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID):
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
else:
# but if local Data already exists, but was not sent - do it now
if remoteData[supplierNum] != 1:
data_sender.A('new-data')
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(
supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID,
):
requests_count += 1
else:
self.missingPackets += 1
else:
# but if local Parity already exists, but was not sent - do it now
if remoteParity[supplierNum] != 1:
data_sender.A('new-data')
total_requests_count += requests_count
if total_requests_count > 0:
lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count)
self.automat('requests-sent', total_requests_count)
else:
if self.missingPackets:
lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets)
self.automat('found-missing')
else:
lg.out(8, 'backup_rebuilder._request_files : nothing was requested')
self.automat('no-requests')
def doDecideToDismiss(self, *args, **kwargs):
"""
Action method.
"""
global _SuppliersToFire
from p2p import p2p_connector
from p2p import network_connector
from customer import supplier_connector
from p2p import online_status
# take any actions only if I am connected to the network
if not p2p_connector.A() or not network_connector.A():
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss p2p_connector() is not ready yet, SKIP'
)
self.automat('made-decision', [])
return
if not network_connector.A():
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss network_connector() is not ready yet, SKIP'
)
self.automat('made-decision', [])
return
if p2p_connector.A().state != 'CONNECTED' or network_connector.A(
).state != 'CONNECTED':
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss p2p/network is not connected at the moment, SKIP'
)
self.automat('made-decision', [])
return
# if certain suppliers needs to be removed by manual/external request just do that
to_be_fired = id_url.to_list(set(_SuppliersToFire))
_SuppliersToFire = []
if to_be_fired:
lg.info('going to fire %d suppliers from external request' %
len(to_be_fired))
self.automat('made-decision', to_be_fired)
return
# make sure to not go too far when i just want to decrease number of my suppliers
number_desired = settings.getSuppliersNumberDesired()
redundant_suppliers = set()
if contactsdb.num_suppliers() > number_desired:
for supplier_index in range(number_desired,
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
lg.info('found REDUNDANT supplier %s at position %d' % (
idurl,
supplier_index,
))
redundant_suppliers.add(idurl)
if redundant_suppliers:
result = list(redundant_suppliers)
lg.info('will replace redundant suppliers: %s' % result)
self.automat('made-decision', result)
return
# now I need to look more careful at my suppliers
potentialy_fired = set()
connected_suppliers = set()
disconnected_suppliers = set()
requested_suppliers = set()
online_suppliers = set()
offline_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
my_suppliers = contactsdb.suppliers()
if _Debug:
lg.args(_DebugLevel, my_suppliers=my_suppliers)
if id_url.is_some_empty(my_suppliers):
lg.warn('SKIP, found empty supplier')
self.automat('made-decision', [])
return
for supplier_idurl in my_suppliers:
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
lg.warn('SKIP, supplier connector for supplier %s not exist' %
supplier_idurl)
continue
if sc.state == 'NO_SERVICE':
lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
disconnected_suppliers.add(supplier_idurl)
potentialy_fired.add(supplier_idurl)
elif sc.state == 'CONNECTED':
connected_suppliers.add(supplier_idurl)
elif sc.state in [
'DISCONNECTED',
'REFUSE',
]:
disconnected_suppliers.add(supplier_idurl)
# elif sc.state in ['QUEUE?', 'REQUEST', ]:
# requested_suppliers.add(supplier_idurl)
if online_status.isOffline(supplier_idurl):
offline_suppliers.add(supplier_idurl)
elif online_status.isOnline(supplier_idurl):
online_suppliers.add(supplier_idurl)
elif online_status.isCheckingNow(supplier_idurl):
requested_suppliers.add(supplier_idurl)
if not connected_suppliers or not online_suppliers:
lg.warn('SKIP, no ONLINE suppliers found at the moment')
self.automat('made-decision', [])
return
if requested_suppliers:
lg.warn('SKIP, still waiting response from some of suppliers')
self.automat('made-decision', [])
return
if not disconnected_suppliers:
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss SKIP, no OFFLINE suppliers found at the moment'
)
# TODO: add more conditions to fire "slow" suppliers - they are still connected but useless
self.automat('made-decision', [])
return
if len(offline_suppliers) + len(online_suppliers) != number_desired:
lg.warn('SKIP, offline + online != total count: %s %s %s' %
(offline_suppliers, online_suppliers, number_desired))
self.automat('made-decision', [])
return
max_offline_suppliers_count = eccmap.GetCorrectableErrors(
number_desired)
if len(offline_suppliers) > max_offline_suppliers_count:
lg.warn(
'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
len(offline_suppliers),
max_offline_suppliers_count,
))
self.automat('made-decision', [])
return
critical_offline_suppliers_count = eccmap.GetFireHireErrors(
number_desired)
if len(offline_suppliers) >= critical_offline_suppliers_count and len(
offline_suppliers) > 0:
if config.conf().getBool(
'services/employer/replace-critically-offline-enabled'):
# TODO: check that issue
# too aggressive replacing suppliers who still have the data is very dangerous !!!
one_dead_supplier = offline_suppliers.pop()
lg.warn(
'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
% (
one_dead_supplier,
critical_offline_suppliers_count,
))
potentialy_fired.add(one_dead_supplier)
if not potentialy_fired:
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!'
)
self.automat('made-decision', [])
return
# only replace suppliers one by one at the moment
result = list(potentialy_fired)
lg.info('will replace supplier %s' % result[0])
self.automat('made-decision', [
result[0],
])
def cmd_suppliers(opts, args, overDict):
def _wait_replace_supplier_and_stop(src, supplier_name, count=0):
suppliers = []
for s in find_comments(src):
if s.count('[online ]') or s.count('[offline]'):
suppliers.append(s[18:38].strip())
if supplier_name not in suppliers:
print ' supplier %s is fired !' % supplier_name
print_and_stop(src)
return
if count >= 60:
print ' time is out\n'
reactor.stop()
return
else:
def _check_again(supplier_name, count):
sys.stdout.write('.')
run_url_command(webcontrol._PAGE_SUPPLIERS).addCallback(_wait_replace_supplier_and_stop, supplier_name, count)
reactor.callLater(1, _check_again, supplier_name, count+1)
if len(args) < 2 or args[1] in [ 'list', 'ls' ]:
url = webcontrol._PAGE_SUPPLIERS
run_url_command(url).addCallback(print_and_stop)
reactor.run()
return 0
elif args[1] in [ 'call', 'cl' ]:
url = webcontrol._PAGE_SUPPLIERS + '?action=call'
run_url_command(url).addCallback(print_and_stop)
reactor.run()
return 0
elif args[1] in [ 'replace', 'rep', 'rp' ] and len(args) >= 3:
contactsdb.init()
idurl = args[2].strip()
if not idurl.startswith('http://'):
try:
idurl = contactsdb.supplier(int(idurl))
except:
idurl = ''
if not idurl:
print 'supplier IDURL is unknown\n'
return 0
name = nameurl.GetName(idurl)
url = webcontrol._PAGE_SUPPLIERS + '?action=replace&idurl=%s' % misc.pack_url_param(idurl)
run_url_command(url).addCallback(_wait_replace_supplier_and_stop, name, 0)
reactor.run()
return 0
elif args[1] in [ 'change', 'ch' ] and len(args) >= 4:
contactsdb.init()
idurl = args[2].strip()
if not idurl.startswith('http://'):
try:
idurl = contactsdb.supplier(int(idurl))
except:
idurl = ''
if not idurl:
print 'supplier IDURL is unknown\n'
return 0
newidurl = args[3].strip()
name = nameurl.GetName(idurl)
newname = nameurl.GetName(newidurl)
url = webcontrol._PAGE_SUPPLIERS + '?action=change&idurl=%s&newidurl=%s' % (misc.pack_url_param(idurl), misc.pack_url_param(newidurl))
run_url_command(url).addCallback(_wait_replace_supplier_and_stop, name, 0)
reactor.run()
return 0
return 2
def doScanAndQueue(self, arg):
global _ShutdownFlag
if _Debug:
lg.out(
_DebugLevel,
'data_sender.doScanAndQueue _ShutdownFlag=%r' % _ShutdownFlag)
if _Debug:
log = open(os.path.join(settings.LogsDir(), 'data_sender.log'),
'w')
log.write('doScanAndQueue %s\n' % time.asctime())
if _ShutdownFlag:
if _Debug:
log.write('doScanAndQueue _ShutdownFlag is True\n')
self.automat('scan-done')
if _Debug:
log.flush()
log.close()
return
for customer_idurl in contactsdb.known_customers():
if '' not in contactsdb.suppliers(customer_idurl):
from storage import backup_matrix
for backupID in misc.sorted_backup_ids(
backup_matrix.local_files().keys(), True):
packetsBySupplier = backup_matrix.ScanBlocksToSend(
backupID)
if _Debug:
log.write('%s\n' % packetsBySupplier)
for supplierNum in packetsBySupplier.keys():
supplier_idurl = contactsdb.supplier(
supplierNum, customer_idurl=customer_idurl)
if not supplier_idurl:
lg.warn('?supplierNum? %s for %s' %
(supplierNum, backupID))
continue
for packetID in packetsBySupplier[supplierNum]:
backupID_, _, supplierNum_, _ = packetid.BidBnSnDp(
packetID)
if backupID_ != backupID:
lg.warn('?backupID? %s for %s' %
(packetID, backupID))
continue
if supplierNum_ != supplierNum:
lg.warn('?supplierNum? %s for %s' %
(packetID, backupID))
continue
if io_throttle.HasPacketInSendQueue(
supplier_idurl, packetID):
if _Debug:
log.write(
'%s already in sending queue for %s\n'
% (packetID, supplier_idurl))
continue
if not io_throttle.OkToSend(supplier_idurl):
if _Debug:
log.write('ok to send %s ? - NO!\n' %
supplier_idurl)
continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
# tranByID = gate.transfers_out_by_idurl().get(supplier_idurl, [])
# if len(tranByID) > 3:
# log.write('transfers by %s: %d\n' % (supplier_idurl, len(tranByID)))
# continue
customerGlobalID, pathID = packetid.SplitPacketID(
packetID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customerGlobalID,
pathID,
)
if not os.path.isfile(filename):
if _Debug:
log.write('%s is not a file\n' % filename)
continue
if io_throttle.QueueSendFile(
filename,
packetID,
supplier_idurl,
my_id.getLocalID(),
self._packetAcked,
self._packetFailed,
):
if _Debug:
log.write(
'io_throttle.QueueSendFile %s\n' %
packetID)
else:
if _Debug:
log.write(
'io_throttle.QueueSendFile FAILED %s\n'
% packetID)
# lg.out(6, ' %s for %s' % (packetID, backupID))
# DEBUG
# break
self.automat('scan-done')
if _Debug:
log.flush()
log.close()