def SupplierSendFile(self, fileName, packetID, ownerID, callOnAck=None, callOnFail=None):
if self.shutdown:
if _Debug:
lg.out(_DebugLevel, "io_throttle.SupplierSendFile finishing to %s, shutdown is True" % self.remoteName)
if callOnFail is not None:
reactor.callLater(0, callOnFail, self.remoteID, packetID, 'shutdown') # @UndefinedVariable
return False
if online_status.isOffline(self.remoteID):
if _Debug:
lg.out(_DebugLevel, "io_throttle.SupplierSendFile %s is offline, so packet %s is failed" % (
self.remoteName, packetID))
if callOnFail is not None:
reactor.callLater(0, callOnFail, self.remoteID, packetID, 'offline') # @UndefinedVariable
return False
if packetID in self.fileSendQueue:
lg.warn("packet %s already in the queue for %s" % (packetID, self.remoteName))
if callOnFail is not None:
reactor.callLater(0, callOnFail, self.remoteID, packetID, 'in queue') # @UndefinedVariable
return False
from stream import file_up
f_up = file_up.FileUp(
self,
fileName,
packetID,
self.remoteID,
ownerID,
callOnAck,
callOnFail,
)
f_up.event('init')
if _Debug:
lg.out(_DebugLevel, "io_throttle.SupplierSendFile %s to %s, %d queued items" % (
packetID, self.remoteName, len(self.fileSendQueue)))
self.DoSend()
return True
def _do_retrieve(self, x=None):
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
localID = my_id.getIDURL()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
pkt_out = p2p_service.SendRetreive(ownerID=localID,
creatorID=localID,
packetID=packetID,
remoteID=supplier_idurl,
response_timeout=60 * 2,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_fail,
})
if pkt_out:
self.requesting_suppliers.add(supplier_idurl)
self.requested_suppliers_number += 1
self.requests_packets_sent.append((packetID, supplier_idurl))
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplier_idurl)))
def doSuppliersRequestIndexFile(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel,
'index_synchronizer.doSuppliersRequestIndexFile')
if driver.is_on('service_backups'):
from storage import backup_control
self.current_local_revision = backup_control.revision()
else:
self.current_local_revision = -1
self.latest_supplier_revision = -1
self.requesting_suppliers.clear()
self.requested_suppliers_number = 0
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
# packetID = settings.BackupIndexFileName()
localID = my_id.getLocalIDURL()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if online_status.isOffline(supplierId):
continue
pkt_out = p2p_service.SendRetreive(localID,
localID,
packetID,
supplierId,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_response,
})
# newpacket = signed.Packet(
# commands.Retrieve(),
# localID,
# localID,
# packetid.RemotePath(packetID),
# '',
# supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Data(): self._on_supplier_response,
# commands.Fail(): self._on_supplier_response, })
if pkt_out:
self.requesting_suppliers.add(supplierId)
self.requested_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def doSuppliersSendIndexFile(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.outgoing_packets_ids = []
self.sent_suppliers_number = 0
localID = my_id.getIDURL()
b = encrypted.Block(
CreatorID=localID,
BackupID=packetID,
BlockNumber=0,
SessionKey=key.NewSessionKey(
session_key_type=key.SessionKeyType()),
SessionKeyType=key.SessionKeyType(),
LastBlock=True,
Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()),
)
Payload = b.Serialize()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplier_idurl,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
if pkt_out:
self.sending_suppliers.add(supplier_idurl)
self.sent_suppliers_number += 1
self.outgoing_packets_ids.append(packetID)
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplier_idurl)))
def doSuppliersSendIndexFile(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
self.sending_suppliers.clear()
self.sent_suppliers_number = 0
localID = my_id.getLocalIDURL()
b = encrypted.Block(
CreatorID=localID,
BackupID=packetID,
BlockNumber=0,
SessionKey=key.NewSessionKey(),
SessionKeyType=key.SessionKeyType(),
LastBlock=True,
Data=bpio.ReadBinaryFile(settings.BackupIndexFilePath()),
)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if online_status.isOffline(supplierId):
continue
newpacket, pkt_out = p2p_service.SendData(
raw_data=Payload,
ownerID=localID,
creatorID=localID,
remoteID=supplierId,
packetID=packetID,
callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked,
},
)
# newpacket = signed.Packet(
# commands.Data(), localID, localID, packetID,
# Payload, supplierId)
# pkt_out = gateway.outbox(newpacket, callbacks={
# commands.Ack(): self._on_supplier_acked,
# commands.Fail(): self._on_supplier_acked, })
if pkt_out:
self.sending_suppliers.add(supplierId)
self.sent_suppliers_number += 1
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(newpacket, nameurl.GetName(supplierId)))
def doDecideToDismiss(self, *args, **kwargs):
"""
Action method.
"""
global _SuppliersToFire
from p2p import p2p_connector
from p2p import network_connector
from customer import supplier_connector
from p2p import online_status
# take any actions only if I am connected to the network
if not p2p_connector.A() or not network_connector.A():
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss p2p_connector() is not ready yet, SKIP'
)
self.automat('made-decision', [])
return
if not network_connector.A():
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss network_connector() is not ready yet, SKIP'
)
self.automat('made-decision', [])
return
if p2p_connector.A().state != 'CONNECTED' or network_connector.A(
).state != 'CONNECTED':
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss p2p/network is not connected at the moment, SKIP'
)
self.automat('made-decision', [])
return
# if certain suppliers needs to be removed by manual/external request just do that
to_be_fired = id_url.to_list(set(_SuppliersToFire))
_SuppliersToFire = []
if to_be_fired:
lg.info('going to fire %d suppliers from external request' %
len(to_be_fired))
self.automat('made-decision', to_be_fired)
return
# make sure to not go too far when i just want to decrease number of my suppliers
number_desired = settings.getSuppliersNumberDesired()
redundant_suppliers = set()
if contactsdb.num_suppliers() > number_desired:
for supplier_index in range(number_desired,
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
lg.info('found REDUNDANT supplier %s at position %d' % (
idurl,
supplier_index,
))
redundant_suppliers.add(idurl)
if redundant_suppliers:
result = list(redundant_suppliers)
lg.info('will replace redundant suppliers: %s' % result)
self.automat('made-decision', result)
return
# now I need to look more careful at my suppliers
potentialy_fired = set()
connected_suppliers = set()
disconnected_suppliers = set()
requested_suppliers = set()
online_suppliers = set()
offline_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
my_suppliers = contactsdb.suppliers()
if _Debug:
lg.args(_DebugLevel, my_suppliers=my_suppliers)
if id_url.is_some_empty(my_suppliers):
lg.warn('SKIP, found empty supplier')
self.automat('made-decision', [])
return
for supplier_idurl in my_suppliers:
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
lg.warn('SKIP, supplier connector for supplier %s not exist' %
supplier_idurl)
continue
if sc.state == 'NO_SERVICE':
lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
disconnected_suppliers.add(supplier_idurl)
potentialy_fired.add(supplier_idurl)
elif sc.state == 'CONNECTED':
connected_suppliers.add(supplier_idurl)
elif sc.state in [
'DISCONNECTED',
'REFUSE',
]:
disconnected_suppliers.add(supplier_idurl)
# elif sc.state in ['QUEUE?', 'REQUEST', ]:
# requested_suppliers.add(supplier_idurl)
if online_status.isOffline(supplier_idurl):
offline_suppliers.add(supplier_idurl)
elif online_status.isOnline(supplier_idurl):
online_suppliers.add(supplier_idurl)
elif online_status.isCheckingNow(supplier_idurl):
requested_suppliers.add(supplier_idurl)
if not connected_suppliers or not online_suppliers:
lg.warn('SKIP, no ONLINE suppliers found at the moment')
self.automat('made-decision', [])
return
if requested_suppliers:
lg.warn('SKIP, still waiting response from some of suppliers')
self.automat('made-decision', [])
return
if not disconnected_suppliers:
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss SKIP, no OFFLINE suppliers found at the moment'
)
# TODO: add more conditions to fire "slow" suppliers - they are still connected but useless
self.automat('made-decision', [])
return
if len(offline_suppliers) + len(online_suppliers) != number_desired:
lg.warn('SKIP, offline + online != total count: %s %s %s' %
(offline_suppliers, online_suppliers, number_desired))
self.automat('made-decision', [])
return
max_offline_suppliers_count = eccmap.GetCorrectableErrors(
number_desired)
if len(offline_suppliers) > max_offline_suppliers_count:
lg.warn(
'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
len(offline_suppliers),
max_offline_suppliers_count,
))
self.automat('made-decision', [])
return
critical_offline_suppliers_count = eccmap.GetFireHireErrors(
number_desired)
if len(offline_suppliers) >= critical_offline_suppliers_count and len(
offline_suppliers) > 0:
if config.conf().getBool(
'services/employer/replace-critically-offline-enabled'):
# TODO: check that issue
# too aggressive replacing suppliers who still have the data is very dangerous !!!
one_dead_supplier = offline_suppliers.pop()
lg.warn(
'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
% (
one_dead_supplier,
critical_offline_suppliers_count,
))
potentialy_fired.add(one_dead_supplier)
if not potentialy_fired:
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!'
)
self.automat('made-decision', [])
return
# only replace suppliers one by one at the moment
result = list(potentialy_fired)
lg.info('will replace supplier %s' % result[0])
self.automat('made-decision', [
result[0],
])
def _do_check_run_requests(self):
if _Debug:
lg.out(
_DebugLevel,
'restore_worker._do_check_run_requests for %s at block %d' % (
self.backup_id,
self.block_number,
))
from stream import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
request_packet_id = packetid.MakePacketID(self.backup_id,
self.block_number,
SupplierNumber, 'Data')
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, OnHandData is True for supplier %d' %
SupplierNumber)
if request_packet_id not in self.block_requests:
self.block_requests[request_packet_id] = True
continue
if request_packet_id in self.block_requests:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, request for packet %r already sent to IO queue for supplier %d'
% (
request_packet_id,
SupplierNumber,
))
continue
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if online_status.isOffline(SupplierID):
if _Debug:
lg.out(_DebugLevel,
' SKIP, offline supplier: %s' % SupplierID)
continue
packetsToRequest.append((
SupplierID,
request_packet_id,
))
for SupplierNumber in range(self.EccMap.paritysegments):
request_packet_id = packetid.MakePacketID(self.backup_id,
self.block_number,
SupplierNumber, 'Parity')
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, OnHandParity is True for supplier %d' %
SupplierNumber)
if request_packet_id not in self.block_requests:
self.block_requests[request_packet_id] = True
continue
if request_packet_id in self.block_requests:
if _Debug:
lg.out(
_DebugLevel,
' SKIP, request for packet %r already sent to IO queue for supplier %d'
% (
request_packet_id,
SupplierNumber,
))
continue
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if online_status.isOffline(SupplierID):
if _Debug:
lg.out(_DebugLevel,
' SKIP, offline supplier: %s' % SupplierID)
continue
packetsToRequest.append((
SupplierID,
request_packet_id,
))
requests_made = 0
# already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
# already_requested += 1
# if packetID not in self.AlreadyRequestedCounts:
# self.AlreadyRequestedCounts[packetID] = 0
# self.AlreadyRequestedCounts[packetID] += 1
lg.warn('packet already in IO queue for supplier %s : %s' % (
SupplierID,
packetID,
))
continue
self.block_requests[packetID] = None
if io_throttle.QueueRequestFile(
callOnReceived=self._on_packet_request_result,
creatorID=self.creator_id,
packetID=packetID,
ownerID=self.creator_id, # self.customer_idurl,
remoteID=SupplierID,
):
requests_made += 1
else:
self.block_requests[packetID] = False
if _Debug:
lg.dbg(
_DebugLevel, 'sent request %r to %r, other requests: %r' %
(packetID, SupplierID, list(self.block_requests.values())))
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel,
" requested %d packets for block %d" % (
requests_made,
self.block_number,
))
return
current_block_requests_results = list(self.block_requests.values())
if _Debug:
lg.args(_DebugLevel,
current_results=current_block_requests_results)
pending_count = current_block_requests_results.count(None)
if pending_count > 0:
if _Debug:
lg.out(
_DebugLevel,
" nothing for request, currently %d pending packets for block %d"
% (
pending_count,
self.block_number,
))
return
failed_count = current_block_requests_results.count(False)
if failed_count > self.max_errors:
lg.err(
'all requests finished and %d packets failed, not possible to read data for block %d'
% (
failed_count,
self.block_number,
))
reactor.callLater(0, self.automat, 'request-failed',
None) # @UndefinedVariable
return
if _Debug:
lg.out(
_DebugLevel,
" all requests finished for block %d : %r" % (
self.block_number,
current_block_requests_results,
))
reactor.callLater(0, self.automat, 'request-finished',
None) # @UndefinedVariable
def doDecideToDismiss(self, *args, **kwargs):
"""
Action method.
"""
global _SuppliersToFire
to_be_fired = list(set(_SuppliersToFire))
_SuppliersToFire = []
if to_be_fired:
lg.warn('going to fire %d suppliers from external request' % len(to_be_fired))
self.automat('made-decision', to_be_fired)
return
potentialy_fired = set()
connected_suppliers = set()
disconnected_suppliers = set()
requested_suppliers = set()
online_suppliers = set()
offline_suppliers = set()
redundant_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers() or None in contactsdb.suppliers() or b'' in contactsdb.suppliers():
lg.warn('SKIP, found empty supplier')
self.automat('made-decision', [])
return
number_desired = settings.getSuppliersNumberDesired()
for supplier_idurl in contactsdb.suppliers():
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
lg.warn('SKIP, supplier connector for supplier %s not exist' % supplier_idurl)
continue
if sc.state == 'NO_SERVICE':
lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
disconnected_suppliers.add(supplier_idurl)
potentialy_fired.add(supplier_idurl)
elif sc.state == 'CONNECTED':
connected_suppliers.add(supplier_idurl)
elif sc.state in [ 'DISCONNECTED', 'REFUSE', ]:
disconnected_suppliers.add(supplier_idurl)
# elif sc.state in ['QUEUE?', 'REQUEST', ]:
# requested_suppliers.add(supplier_idurl)
if online_status.isOffline(supplier_idurl):
offline_suppliers.add(supplier_idurl)
elif online_status.isOnline(supplier_idurl):
online_suppliers.add(supplier_idurl)
elif online_status.isCheckingNow(supplier_idurl):
requested_suppliers.add(supplier_idurl)
if contactsdb.num_suppliers() > number_desired:
for supplier_index in range(number_desired, contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
lg.warn('found "REDUNDANT" supplier %s at position %d' % (
idurl, supplier_index, ))
potentialy_fired.add(idurl)
redundant_suppliers.add(idurl)
else:
lg.warn('supplier at position %d not exist' % supplier_index)
if not connected_suppliers or not online_suppliers:
lg.warn('SKIP, no ONLINE suppliers found at the moment')
self.automat('made-decision', [])
return
if requested_suppliers:
lg.warn('SKIP, still waiting response from some of suppliers')
self.automat('made-decision', [])
return
if redundant_suppliers:
result = list(redundant_suppliers)
lg.info('will replace redundant suppliers: %s' % result)
self.automat('made-decision', result)
return
if not disconnected_suppliers:
lg.warn('SKIP, no OFFLINE suppliers found at the moment')
# TODO: add more conditions to fire "slow" suppliers
self.automat('made-decision', [])
return
if len(offline_suppliers) + len(online_suppliers) != number_desired:
lg.warn('SKIP, offline + online != total count: %s %s %s' % (
offline_suppliers, online_suppliers, number_desired))
self.automat('made-decision', [])
return
max_offline_suppliers_count = eccmap.GetCorrectableErrors(number_desired)
if len(offline_suppliers) > max_offline_suppliers_count:
lg.warn('SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
len(offline_suppliers), max_offline_suppliers_count, ))
self.automat('made-decision', [])
return
critical_offline_suppliers_count = eccmap.GetFireHireErrors(number_desired)
if len(offline_suppliers) >= critical_offline_suppliers_count and len(offline_suppliers) > 0:
# TODO: check that issue
# too aggressive replacing suppliers who still have the data
one_dead_supplier = offline_suppliers.pop()
lg.warn('found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d' % (
one_dead_supplier, critical_offline_suppliers_count, ))
potentialy_fired.add(one_dead_supplier)
if not potentialy_fired:
lg.out(6, 'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!')
self.automat('made-decision', [])
return
# only replace suppliers one by one at the moment
result = list(potentialy_fired)
lg.info('will replace supplier %s' % result[0])
self.automat('made-decision', [result[0], ])
def doRequestPackets(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'restore_worker.doRequestPackets for %s at block %d' % (
self.backup_id,
self.block_number,
))
from customer import io_throttle
packetsToRequest = []
for SupplierNumber in range(self.EccMap.datasegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if online_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandData[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandData is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Data')))
for SupplierNumber in range(self.EccMap.paritysegments):
SupplierID = contactsdb.supplier(
SupplierNumber, customer_idurl=self.customer_idurl)
if not SupplierID:
lg.warn('unknown supplier at position %s' % SupplierNumber)
continue
if online_status.isOffline(SupplierID):
lg.warn('offline supplier: %s' % SupplierID)
continue
if self.OnHandParity[SupplierNumber]:
if _Debug:
lg.out(
_DebugLevel,
' OnHandParity is True for supplier %d' %
SupplierNumber)
continue
packetsToRequest.append(
(SupplierID,
packetid.MakePacketID(self.backup_id, self.block_number,
SupplierNumber, 'Parity')))
if _Debug:
lg.out(_DebugLevel,
' packets to request: %s' % packetsToRequest)
requests_made = 0
already_requested = 0
for SupplierID, packetID in packetsToRequest:
if io_throttle.HasPacketInRequestQueue(SupplierID, packetID):
already_requested += 1
if packetID not in self.AlreadyRequestedCounts:
self.AlreadyRequestedCounts[packetID] = 0
self.AlreadyRequestedCounts[packetID] += 1
if _Debug:
lg.out(
_DebugLevel,
' packet already in request queue: %s %s' % (
SupplierID,
packetID,
))
continue
io_throttle.QueueRequestFile(self._on_packet_request_result,
self.creator_id, packetID,
self.creator_id, SupplierID)
requests_made += 1
del packetsToRequest
if requests_made:
if _Debug:
lg.out(
_DebugLevel, " requested %d packets for block %d" %
(requests_made, self.block_number))
else:
if not already_requested:
lg.warn('no requests made for block %d' % self.block_number)
self.automat('request-failed', None)
else:
if _Debug:
lg.out(
_DebugLevel,
" found %d already requested packets for block %d"
% (already_requested, self.block_number))
if self.AlreadyRequestedCounts:
all_counts = sorted(self.AlreadyRequestedCounts.values())
if all_counts[0] > 100:
lg.warn('too much requests made for block %d' %
self.block_number)
self.automat('request-failed', None)