def doRequestPackets(self, arg):
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
continue
if not self.OnHandData[SupplierNumber] and contact_status.isOnline(
SupplierID):
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(SupplierNumber,
customer_idurl=self.CustomerIDURL)
if not SupplierID:
continue
if not self.OnHandParity[
SupplierNumber] and contact_status.isOnline(SupplierID):
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.BackupID, self.BlockNumber,
SupplierNumber, 'Parity')))
for SupplierID, packetID in packetsToRequest:
io_throttle.QueueRequestFile(self._on_inbox_packet, self.CreatorID,
packetID, self.CreatorID, SupplierID)
lg.out(
6, "restore.doRequestPackets requested %d packets for block %d" %
(len(packetsToRequest), self.BlockNumber))
del packetsToRequest
self.automat('request-done')
def doSuppliersSendDBInfo(self, arg):
from p2p import contact_status
lg.out(4, 'backup_db_keeper.doSuppliersSendDBInfo')
packetID = settings.BackupIndexFileName()
self.sentSuppliers.clear()
src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = my_id.getLocalID()
b = encrypted.Block(
localID,
packetID,
0,
key.NewSessionKey(),
key.SessionKeyType(),
True,
src)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = signed.Packet(
commands.Data(), localID, localID, packetID,
Payload, supplierId)
pkt_out = gateway.outbox(newpacket, callbacks={
commands.Ack(): self._supplier_acked,
commands.Fail(): self._supplier_acked, })
self.sentSuppliers.add(supplierId)
lg.out(
4, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def send_message(json_data, recipient_global_id, packet_id=None, timeout=None):
"""
Send command.Message() packet to remote peer.
Returns Deferred (if remote_idurl was not cached yet) or outbox packet object.
"""
global _LastUserPingTime
global _PingTrustIntervalSeconds
if not packet_id:
packet_id = packetid.UniqueID()
lg.out(
4, "message.send_message to %s with PackteID=%s" %
(recipient_global_id, packet_id))
remote_idurl = global_id.GlobalUserToIDURL(recipient_global_id)
if not remote_idurl:
return fail(Exception('invalid recipient'))
ret = Deferred()
if remote_idurl not in _LastUserPingTime:
is_expired = True
else:
is_expired = time.time(
) - _LastUserPingTime[remote_idurl] > _PingTrustIntervalSeconds
remote_identity = identitycache.FromCache(remote_idurl)
if is_expired or remote_identity is None or not contact_status.isOnline(
remote_idurl):
d = propagate.PingContact(remote_idurl, timeout=timeout or 5)
d.addCallback(lambda response_tuple: on_ping_success(
response_tuple, remote_idurl))
d.addCallback(
lambda response_tuple: do_send_message(json_data,
recipient_global_id,
packet_id,
timeout,
result_defer=ret))
d.addErrback(lambda err: on_message_failed(remote_idurl,
json_data,
recipient_global_id,
packet_id,
None,
None,
result_defer=ret))
return ret
try:
do_send_message(json_data, recipient_global_id, packet_id, timeout,
ret)
except Exception as exc:
lg.warn(str(exc))
on_message_failed(
remote_idurl,
json_data,
recipient_global_id,
packet_id,
None,
None,
)
ret.errback(exc)
return ret
def doSuppliersRequestIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel,
'index_synchronizer.doSuppliersRequestIndexFile')
if driver.is_on('service_backups'):
from storage import backup_control
self.current_local_revision = backup_control.revision()
else:
self.current_local_revision = -1
self.latest_supplier_revision = -1
self.requesting_suppliers.clear()
self.requested_suppliers_number = 0
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
# packetID = settings.BackupIndexFileName()
localID = my_id.getLocalID()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
pkt_out = p2p_service.SendRetreive(localID,
localID,
packetID,
supplierId,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_response,
})
# newpacket = signed.Packet(
# commands.Retrieve(),
# localID,
# localID,
# packetid.RemotePath(packetID),
# '',
# supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Data(): self._on_supplier_response,
# commands.Fail(): self._on_supplier_response, })
if pkt_out:
self.requesting_suppliers.add(supplierId)
self.requested_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def doSuppliersSendIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.sent_suppliers_number = 0
src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = my_id.getLocalID()
b = encrypted.Block(
localID,
packetID,
0,
key.NewSessionKey(),
key.SessionKeyType(),
True,
src,
)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplierId,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
# newpacket = signed.Packet(
# commands.Data(), localID, localID, packetID,
# Payload, supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Ack(): self._on_supplier_acked,
# commands.Fail(): self._on_supplier_acked, })
if pkt_out:
self.sending_suppliers.add(supplierId)
self.sent_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplierId)))
def isLineActive(self, arg):
"""
Condition method.
"""
for idurl in self.connected_broadcasters:
if contact_status.isOnline(idurl):
# if at least one broadcaster is connected
# we assume the line is still active
return True
if not self.last_success_action_time:
return False
return time.time() - self.last_success_action_time > 5 * 60
def doRequestRemoteFiles(self, arg):
global _RequestedListFilesCounter
global _RequestedListFilesPacketIDs
_RequestedListFilesCounter = 0
_RequestedListFilesPacketIDs.clear()
for idurl in contactsdb.suppliers():
if idurl:
if contact_status.isOnline(idurl):
p2p_service.SendRequestListFiles(idurl)
_RequestedListFilesPacketIDs.add(idurl)
else:
lg.out(6, 'list_files_orator.doRequestRemoteFiles SKIP %s is not online' % idurl)
def doRequestRemoteFiles(self, arg):
global _RequestedListFilesCounter
global _RequestedListFilesPacketIDs
_RequestedListFilesCounter = 0
_RequestedListFilesPacketIDs.clear()
for idurl in contactsdb.suppliers():
if idurl:
if contact_status.isOnline(idurl):
p2p_service.SendRequestListFiles(idurl)
_RequestedListFilesPacketIDs.add(idurl)
else:
lg.out(6, 'list_files_orator.doRequestRemoteFiles SKIP %s is not online' % idurl)
def isLineActive(self, arg):
"""
Condition method.
"""
for idurl in self.connected_broadcasters:
if contact_status.isOnline(idurl):
# if at least one broadcaster is connected
# we assume the line is still active
return True
if not self.last_success_action_time:
return False
return time.time() - self.last_success_action_time > 5 * 60
def rate_all_users():
lg.out(4, 'ratings.rate_all_users')
monthStr = time.strftime('%B')
from p2p import contact_status
for idurl in contactsdb.contacts_full():
isalive = contact_status.isOnline(idurl)
mall, malive, tall, talive = increase_rating(idurl, isalive)
month_percent = 100.0 * float(malive) / float(mall)
total_percent = 100.0 * float(talive) / float(tall)
lg.out(4, '[%6.2f%%: %s/%s] in %s and [%6.2f%%: %s/%s] total - %s' % (
month_percent,
malive,
mall,
monthStr,
total_percent,
talive,
tall,
nameurl.GetName(idurl),))
read_index()
def GetActiveArray():
"""
Loops all suppliers and returns who is alive at the moment.
Return a list with integers: 0 for offline suppler and 1 if he is
available right now. Uses ``p2p.contact_status.isOnline()`` to see
the current state of supplier.
"""
from p2p import contact_status
activeArray = [0] * contactsdb.num_suppliers()
for i in xrange(contactsdb.num_suppliers()):
suplier_idurl = contactsdb.supplier(i)
if not suplier_idurl:
continue
if contact_status.isOnline(suplier_idurl):
activeArray[i] = 1
else:
activeArray[i] = 0
return activeArray
def doSuppliersSendIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = settings.BackupIndexFileName()
self.sending_suppliers.clear()
self.sent_suppliers_number = 0
src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = my_id.getLocalID()
b = encrypted.Block(
localID,
packetID,
0,
key.NewSessionKey(),
key.SessionKeyType(),
True,
src)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = signed.Packet(
commands.Data(), localID, localID, packetID,
Payload, supplierId)
pkt_out = gateway.outbox(newpacket, callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked, })
if pkt_out:
self.sending_suppliers.add(supplierId)
self.sent_suppliers_number += 1
if _Debug:
lg.out(_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def doSuppliersRequestIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersRequestIndexFile')
if driver.is_started('service_backups'):
from storage import backup_control
self.current_local_revision = backup_control.revision()
else:
self.current_local_revision = -1
self.latest_supplier_revision = -1
self.requesting_suppliers.clear()
self.requested_suppliers_number = 0
packetID = settings.BackupIndexFileName()
localID = my_id.getLocalID()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = signed.Packet(
commands.Retrieve(),
localID,
localID,
packetID,
'',
supplierId)
pkt_out = gateway.outbox(newpacket, callbacks={
commands.Data(): self._on_supplier_response,
commands.Fail(): self._on_supplier_response, })
if pkt_out:
self.requesting_suppliers.add(supplierId)
self.requested_suppliers_number += 1
if _Debug:
lg.out(_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
to_be_fired = list(set(_SuppliersToFire))
_SuppliersToFire = []
if to_be_fired:
lg.warn('going to fire %d suppliers from external request' %
len(to_be_fired))
self.automat('made-decision', to_be_fired)
return
potentialy_fired = set()
connected_suppliers = set()
disconnected_suppliers = set()
requested_suppliers = set()
online_suppliers = set()
offline_suppliers = set()
redundant_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers() or None in contactsdb.suppliers():
lg.warn('SKIP, found empty supplier')
self.automat('made-decision', [])
return
number_desired = settings.getSuppliersNumberDesired()
for supplier_idurl in contactsdb.suppliers():
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
lg.warn('SKIP, supplier connector for supplier %s not exist' %
supplier_idurl)
continue
if sc.state == 'NO_SERVICE':
lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
disconnected_suppliers.add(supplier_idurl)
potentialy_fired.add(supplier_idurl)
elif sc.state == 'CONNECTED':
connected_suppliers.add(supplier_idurl)
elif sc.state in [
'DISCONNECTED',
'REFUSE',
]:
disconnected_suppliers.add(supplier_idurl)
# elif sc.state in ['QUEUE?', 'REQUEST', ]:
# requested_suppliers.add(supplier_idurl)
if contact_status.isOffline(supplier_idurl):
offline_suppliers.add(supplier_idurl)
elif contact_status.isOnline(supplier_idurl):
online_suppliers.add(supplier_idurl)
elif contact_status.isCheckingNow(supplier_idurl):
requested_suppliers.add(supplier_idurl)
if contactsdb.num_suppliers() > number_desired:
for supplier_index in range(number_desired,
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
lg.warn('found "REDUNDANT" supplier %s at position %d' % (
idurl,
supplier_index,
))
potentialy_fired.add(idurl)
redundant_suppliers.add(idurl)
else:
lg.warn('supplier at position %d not exist' %
supplier_index)
if not connected_suppliers or not online_suppliers:
lg.warn('SKIP, no ONLINE suppliers found at the moment')
self.automat('made-decision', [])
return
if requested_suppliers:
lg.warn('SKIP, still waiting response from some of suppliers')
self.automat('made-decision', [])
return
if redundant_suppliers:
result = list(redundant_suppliers)
lg.info('will replace redundant suppliers: %s' % result)
self.automat('made-decision', result)
return
if not disconnected_suppliers:
lg.warn('SKIP, no OFFLINE suppliers found at the moment')
# TODO: add more conditions to fire "slow" suppliers
self.automat('made-decision', [])
return
if len(offline_suppliers) + len(online_suppliers) != number_desired:
lg.warn('SKIP, offline + online != total count: %s %s %s' %
(offline_suppliers, online_suppliers, number_desired))
self.automat('made-decision', [])
return
from raid import eccmap
max_offline_suppliers_count = eccmap.GetCorrectableErrors(
number_desired)
if len(offline_suppliers) > max_offline_suppliers_count:
lg.warn(
'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
len(offline_suppliers),
max_offline_suppliers_count,
))
self.automat('made-decision', [])
return
critical_offline_suppliers_count = eccmap.GetFireHireErrors(
number_desired)
# TODO: temporary disabled because of an issue: too aggressive replacing suppliers who still have the data
if False: # len(offline_suppliers) >= critical_offline_suppliers_count:
one_dead_supplier = offline_suppliers.pop()
lg.warn(
'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
% (
one_dead_supplier,
critical_offline_suppliers_count,
))
potentialy_fired.add(one_dead_supplier)
if not potentialy_fired:
lg.out(
6,
'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!'
)
self.automat('made-decision', [])
return
# only replace suppliers one by one at the moment
result = list(potentialy_fired)
lg.info('will replace supplier %s' % result[0])
self.automat('made-decision', [
result[0],
])
