示例#1
0
 def doRequestPackets(self, arg):
     from customer import io_throttle
     packetsToRequest = []
     for SupplierNumber in range(self.EccMap.datasegments):
         SupplierID = contactsdb.supplier(SupplierNumber,
                                          customer_idurl=self.CustomerIDURL)
         if not SupplierID:
             continue
         if not self.OnHandData[SupplierNumber] and contact_status.isOnline(
                 SupplierID):
             packetsToRequest.append(
                 (SupplierID,
                  packetid.MakePacketID(self.BackupID, self.BlockNumber,
                                        SupplierNumber, 'Data')))
     for SupplierNumber in range(self.EccMap.paritysegments):
         SupplierID = contactsdb.supplier(SupplierNumber,
                                          customer_idurl=self.CustomerIDURL)
         if not SupplierID:
             continue
         if not self.OnHandParity[
                 SupplierNumber] and contact_status.isOnline(SupplierID):
             packetsToRequest.append(
                 (SupplierID,
                  packetid.MakePacketID(self.BackupID, self.BlockNumber,
                                        SupplierNumber, 'Parity')))
     for SupplierID, packetID in packetsToRequest:
         io_throttle.QueueRequestFile(self._on_inbox_packet, self.CreatorID,
                                      packetID, self.CreatorID, SupplierID)
     lg.out(
         6, "restore.doRequestPackets requested %d packets for block %d" %
         (len(packetsToRequest), self.BlockNumber))
     del packetsToRequest
     self.automat('request-done')
 def doSuppliersSendDBInfo(self, arg):
     from p2p import contact_status
     lg.out(4, 'backup_db_keeper.doSuppliersSendDBInfo')
     packetID = settings.BackupIndexFileName()
     self.sentSuppliers.clear()
     src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
     localID = my_id.getLocalID()
     b = encrypted.Block(
         localID,
         packetID,
         0,
         key.NewSessionKey(),
         key.SessionKeyType(),
         True,
         src)
     Payload = b.Serialize()
     for supplierId in contactsdb.suppliers():
         if not supplierId:
             continue
         if not contact_status.isOnline(supplierId):
             continue
         newpacket = signed.Packet(
             commands.Data(), localID, localID, packetID,
             Payload, supplierId)
         pkt_out = gateway.outbox(newpacket, callbacks={
             commands.Ack(): self._supplier_acked,
             commands.Fail(): self._supplier_acked, })
         self.sentSuppliers.add(supplierId)
         lg.out(
             4, '    %s sending to %s' %
             (pkt_out, nameurl.GetName(supplierId)))
示例#3
0
def send_message(json_data, recipient_global_id, packet_id=None, timeout=None):
    """
    Send command.Message() packet to remote peer.
    Returns Deferred (if remote_idurl was not cached yet) or outbox packet object.
    """
    global _LastUserPingTime
    global _PingTrustIntervalSeconds
    if not packet_id:
        packet_id = packetid.UniqueID()
    lg.out(
        4, "message.send_message to %s with PackteID=%s" %
        (recipient_global_id, packet_id))
    remote_idurl = global_id.GlobalUserToIDURL(recipient_global_id)
    if not remote_idurl:
        return fail(Exception('invalid recipient'))
    ret = Deferred()
    if remote_idurl not in _LastUserPingTime:
        is_expired = True
    else:
        is_expired = time.time(
        ) - _LastUserPingTime[remote_idurl] > _PingTrustIntervalSeconds
    remote_identity = identitycache.FromCache(remote_idurl)
    if is_expired or remote_identity is None or not contact_status.isOnline(
            remote_idurl):
        d = propagate.PingContact(remote_idurl, timeout=timeout or 5)
        d.addCallback(lambda response_tuple: on_ping_success(
            response_tuple, remote_idurl))
        d.addCallback(
            lambda response_tuple: do_send_message(json_data,
                                                   recipient_global_id,
                                                   packet_id,
                                                   timeout,
                                                   result_defer=ret))
        d.addErrback(lambda err: on_message_failed(remote_idurl,
                                                   json_data,
                                                   recipient_global_id,
                                                   packet_id,
                                                   None,
                                                   None,
                                                   result_defer=ret))
        return ret
    try:
        do_send_message(json_data, recipient_global_id, packet_id, timeout,
                        ret)
    except Exception as exc:
        lg.warn(str(exc))
        on_message_failed(
            remote_idurl,
            json_data,
            recipient_global_id,
            packet_id,
            None,
            None,
        )
        ret.errback(exc)
    return ret
示例#4
0
 def doSuppliersRequestIndexFile(self, arg):
     """
     Action method.
     """
     if _Debug:
         lg.out(_DebugLevel,
                'index_synchronizer.doSuppliersRequestIndexFile')
     if driver.is_on('service_backups'):
         from storage import backup_control
         self.current_local_revision = backup_control.revision()
     else:
         self.current_local_revision = -1
     self.latest_supplier_revision = -1
     self.requesting_suppliers.clear()
     self.requested_suppliers_number = 0
     packetID = global_id.MakeGlobalID(
         customer=my_id.getGlobalID(key_alias='master'),
         path=settings.BackupIndexFileName(),
     )
     # packetID = settings.BackupIndexFileName()
     localID = my_id.getLocalID()
     for supplierId in contactsdb.suppliers():
         if not supplierId:
             continue
         if not contact_status.isOnline(supplierId):
             continue
         pkt_out = p2p_service.SendRetreive(localID,
                                            localID,
                                            packetID,
                                            supplierId,
                                            callbacks={
                                                commands.Data():
                                                self._on_supplier_response,
                                                commands.Fail():
                                                self._on_supplier_response,
                                            })
         #             newpacket = signed.Packet(
         #                 commands.Retrieve(),
         #                 localID,
         #                 localID,
         #                 packetid.RemotePath(packetID),
         #                 '',
         #                 supplierId)
         #             pkt_out = gateway.outbox(newpacket, callbacks={
         #                 commands.Data(): self._on_supplier_response,
         #                 commands.Fail(): self._on_supplier_response, })
         if pkt_out:
             self.requesting_suppliers.add(supplierId)
             self.requested_suppliers_number += 1
         if _Debug:
             lg.out(
                 _DebugLevel, '    %s sending to %s' %
                 (pkt_out, nameurl.GetName(supplierId)))
 def doSuppliersSendIndexFile(self, arg):
     """
     Action method.
     """
     if _Debug:
         lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
     packetID = global_id.MakeGlobalID(
         customer=my_id.getGlobalID(key_alias='master'),
         path=settings.BackupIndexFileName(),
     )
     self.sending_suppliers.clear()
     self.sent_suppliers_number = 0
     src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
     localID = my_id.getLocalID()
     b = encrypted.Block(
         localID,
         packetID,
         0,
         key.NewSessionKey(),
         key.SessionKeyType(),
         True,
         src,
     )
     Payload = b.Serialize()
     for supplierId in contactsdb.suppliers():
         if not supplierId:
             continue
         if not contact_status.isOnline(supplierId):
             continue
         newpacket, pkt_out = p2p_service.SendData(
             raw_data=Payload,
             ownerID=localID,
             creatorID=localID,
             remoteID=supplierId,
             packetID=packetID,
             callbacks={
                 commands.Ack(): self._on_supplier_acked,
                 commands.Fail(): self._on_supplier_acked,
             },
         )
         # newpacket = signed.Packet(
         #     commands.Data(), localID, localID, packetID,
         #     Payload, supplierId)
         # pkt_out = gateway.outbox(newpacket, callbacks={
         #     commands.Ack(): self._on_supplier_acked,
         #     commands.Fail(): self._on_supplier_acked, })
         if pkt_out:
             self.sending_suppliers.add(supplierId)
             self.sent_suppliers_number += 1
         if _Debug:
             lg.out(
                 _DebugLevel, '    %s sending to %s' %
                 (newpacket, nameurl.GetName(supplierId)))
示例#6
0
 def isLineActive(self, arg):
     """
     Condition method.
     """
     for idurl in self.connected_broadcasters:
         if contact_status.isOnline(idurl):
             # if at least one broadcaster is connected
             # we assume the line is still active
             return True
     if not self.last_success_action_time:
         return False
     return time.time() - self.last_success_action_time > 5 * 60
示例#7
0
 def doRequestRemoteFiles(self, arg):
     global _RequestedListFilesCounter
     global _RequestedListFilesPacketIDs
     _RequestedListFilesCounter = 0
     _RequestedListFilesPacketIDs.clear()
     for idurl in contactsdb.suppliers():
         if idurl:
             if contact_status.isOnline(idurl):
                 p2p_service.SendRequestListFiles(idurl)
                 _RequestedListFilesPacketIDs.add(idurl)
             else:
                 lg.out(6, 'list_files_orator.doRequestRemoteFiles SKIP %s is not online' % idurl)
 def doRequestRemoteFiles(self, arg):
     global _RequestedListFilesCounter
     global _RequestedListFilesPacketIDs
     _RequestedListFilesCounter = 0
     _RequestedListFilesPacketIDs.clear()
     for idurl in contactsdb.suppliers():
         if idurl:
             if contact_status.isOnline(idurl):
                 p2p_service.SendRequestListFiles(idurl)
                 _RequestedListFilesPacketIDs.add(idurl)
             else:
                 lg.out(6, 'list_files_orator.doRequestRemoteFiles SKIP %s is not online' % idurl)
 def isLineActive(self, arg):
     """
     Condition method.
     """
     for idurl in self.connected_broadcasters:
         if contact_status.isOnline(idurl):
             # if at least one broadcaster is connected
             # we assume the line is still active
             return True
     if not self.last_success_action_time:
         return False
     return time.time() - self.last_success_action_time > 5 * 60
示例#10
0
def rate_all_users():
    lg.out(4, 'ratings.rate_all_users')
    monthStr = time.strftime('%B')
    from p2p import contact_status
    for idurl in contactsdb.contacts_full():
        isalive = contact_status.isOnline(idurl)
        mall, malive, tall, talive = increase_rating(idurl, isalive)
        month_percent = 100.0 * float(malive) / float(mall)
        total_percent = 100.0 * float(talive) / float(tall)
        lg.out(4, '[%6.2f%%: %s/%s] in %s and [%6.2f%%: %s/%s] total - %s' % (
            month_percent,
            malive,
            mall,
            monthStr,
            total_percent,
            talive,
            tall,
            nameurl.GetName(idurl),))
    read_index()
示例#11
0
def GetActiveArray():
    """
    Loops all suppliers and returns who is alive at the moment.

    Return a list with integers: 0 for offline suppler and 1 if he is
    available right now. Uses ``p2p.contact_status.isOnline()`` to see
    the current state of supplier.
    """
    from p2p import contact_status

    activeArray = [0] * contactsdb.num_suppliers()
    for i in xrange(contactsdb.num_suppliers()):
        suplier_idurl = contactsdb.supplier(i)
        if not suplier_idurl:
            continue
        if contact_status.isOnline(suplier_idurl):
            activeArray[i] = 1
        else:
            activeArray[i] = 0
    return activeArray
 def doSuppliersSendIndexFile(self, arg):
     """
     Action method.
     """
     if _Debug:
         lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
     packetID = settings.BackupIndexFileName()
     self.sending_suppliers.clear()
     self.sent_suppliers_number = 0
     src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
     localID = my_id.getLocalID()
     b = encrypted.Block(
         localID,
         packetID,
         0,
         key.NewSessionKey(),
         key.SessionKeyType(),
         True,
         src)
     Payload = b.Serialize()
     for supplierId in contactsdb.suppliers():
         if not supplierId:
             continue
         if not contact_status.isOnline(supplierId):
             continue
         newpacket = signed.Packet(
             commands.Data(), localID, localID, packetID,
             Payload, supplierId)
         pkt_out = gateway.outbox(newpacket, callbacks={
             commands.Ack(): self._on_supplier_acked,
             commands.Fail(): self._on_supplier_acked, })
         if pkt_out:
             self.sending_suppliers.add(supplierId)
             self.sent_suppliers_number += 1
         if _Debug:
             lg.out(_DebugLevel, '    %s sending to %s' %
                    (pkt_out, nameurl.GetName(supplierId)))
 def doSuppliersRequestIndexFile(self, arg):
     """
     Action method.
     """
     if _Debug:
         lg.out(_DebugLevel, 'index_synchronizer.doSuppliersRequestIndexFile')
     if driver.is_started('service_backups'):
         from storage import backup_control
         self.current_local_revision = backup_control.revision()
     else:
         self.current_local_revision = -1
     self.latest_supplier_revision = -1
     self.requesting_suppliers.clear()
     self.requested_suppliers_number = 0
     packetID = settings.BackupIndexFileName()
     localID = my_id.getLocalID()
     for supplierId in contactsdb.suppliers():
         if not supplierId:
             continue
         if not contact_status.isOnline(supplierId):
             continue
         newpacket = signed.Packet(
             commands.Retrieve(),
             localID,
             localID,
             packetID,
             '',
             supplierId)
         pkt_out = gateway.outbox(newpacket, callbacks={
             commands.Data(): self._on_supplier_response,
             commands.Fail(): self._on_supplier_response, })
         if pkt_out:
             self.requesting_suppliers.add(supplierId)
             self.requested_suppliers_number += 1
         if _Debug:
             lg.out(_DebugLevel, '    %s sending to %s' %
                    (pkt_out, nameurl.GetName(supplierId)))
示例#14
0
    def doDecideToDismiss(self, arg):
        """
        Action method.
        """
        global _SuppliersToFire
        to_be_fired = list(set(_SuppliersToFire))
        _SuppliersToFire = []
        if to_be_fired:
            lg.warn('going to fire %d suppliers from external request' %
                    len(to_be_fired))
            self.automat('made-decision', to_be_fired)
            return
        potentialy_fired = set()
        connected_suppliers = set()
        disconnected_suppliers = set()
        requested_suppliers = set()
        online_suppliers = set()
        offline_suppliers = set()
        redundant_suppliers = set()
        # if you have some empty suppliers need to get rid of them,
        # but no need to dismiss anyone at the moment.
        if '' in contactsdb.suppliers() or None in contactsdb.suppliers():
            lg.warn('SKIP, found empty supplier')
            self.automat('made-decision', [])
            return
        number_desired = settings.getSuppliersNumberDesired()
        for supplier_idurl in contactsdb.suppliers():
            sc = supplier_connector.by_idurl(supplier_idurl)
            if not sc:
                lg.warn('SKIP, supplier connector for supplier %s not exist' %
                        supplier_idurl)
                continue
            if sc.state == 'NO_SERVICE':
                lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
                disconnected_suppliers.add(supplier_idurl)
                potentialy_fired.add(supplier_idurl)
            elif sc.state == 'CONNECTED':
                connected_suppliers.add(supplier_idurl)
            elif sc.state in [
                    'DISCONNECTED',
                    'REFUSE',
            ]:
                disconnected_suppliers.add(supplier_idurl)
#             elif sc.state in ['QUEUE?', 'REQUEST', ]:
#                 requested_suppliers.add(supplier_idurl)
            if contact_status.isOffline(supplier_idurl):
                offline_suppliers.add(supplier_idurl)
            elif contact_status.isOnline(supplier_idurl):
                online_suppliers.add(supplier_idurl)
            elif contact_status.isCheckingNow(supplier_idurl):
                requested_suppliers.add(supplier_idurl)
        if contactsdb.num_suppliers() > number_desired:
            for supplier_index in range(number_desired,
                                        contactsdb.num_suppliers()):
                idurl = contactsdb.supplier(supplier_index)
                if idurl:
                    lg.warn('found "REDUNDANT" supplier %s at position %d' % (
                        idurl,
                        supplier_index,
                    ))
                    potentialy_fired.add(idurl)
                    redundant_suppliers.add(idurl)
                else:
                    lg.warn('supplier at position %d not exist' %
                            supplier_index)
        if not connected_suppliers or not online_suppliers:
            lg.warn('SKIP, no ONLINE suppliers found at the moment')
            self.automat('made-decision', [])
            return
        if requested_suppliers:
            lg.warn('SKIP, still waiting response from some of suppliers')
            self.automat('made-decision', [])
            return
        if redundant_suppliers:
            result = list(redundant_suppliers)
            lg.info('will replace redundant suppliers: %s' % result)
            self.automat('made-decision', result)
            return
        if not disconnected_suppliers:
            lg.warn('SKIP, no OFFLINE suppliers found at the moment')
            # TODO: add more conditions to fire "slow" suppliers
            self.automat('made-decision', [])
            return
        if len(offline_suppliers) + len(online_suppliers) != number_desired:
            lg.warn('SKIP, offline + online != total count: %s %s %s' %
                    (offline_suppliers, online_suppliers, number_desired))
            self.automat('made-decision', [])
            return
        from raid import eccmap
        max_offline_suppliers_count = eccmap.GetCorrectableErrors(
            number_desired)
        if len(offline_suppliers) > max_offline_suppliers_count:
            lg.warn(
                'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
                    len(offline_suppliers),
                    max_offline_suppliers_count,
                ))
            self.automat('made-decision', [])
            return
        critical_offline_suppliers_count = eccmap.GetFireHireErrors(
            number_desired)
        # TODO:  temporary disabled because of an issue: too aggressive replacing suppliers who still have the data
        if False:  # len(offline_suppliers) >= critical_offline_suppliers_count:
            one_dead_supplier = offline_suppliers.pop()
            lg.warn(
                'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
                % (
                    one_dead_supplier,
                    critical_offline_suppliers_count,
                ))
            potentialy_fired.add(one_dead_supplier)
        if not potentialy_fired:
            lg.out(
                6,
                'fire_hire.doDecideToDismiss   found no "bad" suppliers, all is good !!!!!'
            )
            self.automat('made-decision', [])
            return
        # only replace suppliers one by one at the moment
        result = list(potentialy_fired)
        lg.info('will replace supplier %s' % result[0])
        self.automat('made-decision', [
            result[0],
        ])