def OnFileSendAckReceived(self, newpacket, info):
if self.shutdown:
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.OnFileSendAckReceived finishing to %s, shutdown is True"
% self.remoteName)
return
if not newpacket and not info:
lg.warn('packet timed out during responding')
return
self.ackedCount += 1
packetID = global_id.CanonicalID(newpacket.PacketID)
if packetID not in self.fileSendQueue:
lg.warn("packet %s not in sending queue for %s" %
(newpacket.PacketID, self.remoteName))
return
if packetID not in list(self.fileSendDict.keys()):
lg.warn("packet %s not in sending dict for %s" %
(newpacket.PacketID, self.remoteName))
return
self.fileSendDict[packetID].ackTime = time.time()
if newpacket.Command == commands.Ack():
self.fileSendDict[packetID].result = 'acked'
if self.fileSendDict[packetID].callOnAck:
reactor.callLater(0, self.fileSendDict[packetID].callOnAck,
newpacket, newpacket.OwnerID,
packetID) # @UndefinedVariable
elif newpacket.Command == commands.Fail():
self.fileSendDict[packetID].result = 'failed'
if self.fileSendDict[packetID].callOnFail:
reactor.callLater(0, self.fileSendDict[packetID].callOnFail,
newpacket.CreatorID, packetID,
'failed') # @UndefinedVariable
from customer import supplier_connector
sc = supplier_connector.by_idurl(newpacket.OwnerID)
if sc:
if newpacket.Command == commands.Ack():
sc.automat('ack', newpacket)
elif newpacket.Command == commands.Fail():
sc.automat('fail', newpacket)
# elif newpacket.Command == commands.Data():
# sc.automat('data', newpacket)
else:
raise Exception('incorrect packet type received')
self.DoSend()
# self.RunSend()
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.OnFileSendAckReceived %s from %s, queue=%d" %
(str(newpacket), self.remoteName, len(self.fileSendQueue)))
def doSendMyIdentity(self, *args, **kwargs):
"""
Action method.
"""
global _KnownChannels
self.ping_attempts += 1
if self.fake_identity:
identity_object = self.fake_identity
else:
identity_object = my_id.getLocalIdentity()
if not identity_object.Valid():
raise Exception('can not use invalid identity for ping')
if self.channel_counter:
packet_id = '%s:%d:%d:%s' % (self.channel, _KnownChannels[self.channel], self.ping_attempts, packetid.UniqueID())
else:
packet_id = '%s:%d:%s' % (self.channel, self.ping_attempts, packetid.UniqueID())
ping_packet = signed.Packet(
Command=commands.Identity(),
OwnerID=my_id.getIDURL(),
CreatorID=my_id.getIDURL(),
PacketID=packet_id,
Payload=strng.to_bin(identity_object.serialize()),
RemoteID=self.remote_idurl,
)
if self.skip_outbox:
packet_out.create(
outpacket=ping_packet,
wide=True,
response_timeout=self.ack_timeout,
callbacks={
commands.Ack(): lambda response, info: self.automat('ack-received', response=response, info=info),
commands.Fail(): lambda response, info: self.automat('fail-received', response=response, info=info),
None: lambda pkt_out: self.automat('ack-timeout', pkt_out),
},
keep_alive=self.keep_alive,
)
else:
gateway.outbox(
outpacket=ping_packet,
wide=True,
response_timeout=self.ack_timeout,
callbacks={
commands.Ack(): lambda response, info: self.automat('ack-received', response=response, info=info),
commands.Fail(): lambda response, info: self.automat('fail-received', response=response, info=info),
None: lambda pkt_out: self.automat('ack-timeout', pkt_out),
},
keep_alive=self.keep_alive,
)
if _Debug:
lg.args(_DebugLevel, packet_id=packet_id, remote_idurl=self.remote_idurl, ping_attempts=self.ping_attempts)
def OnFileSendAckReceived(self, newpacket, info):
if self.shutdown:
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.OnFileSendAckReceived finishing to %s, shutdown is True"
% self.remoteName)
return
if not newpacket and not info:
lg.warn('packet timed out during responding')
return
if _Debug:
lg.out(_DebugLevel,
"io_throttle.OnFileSendAckReceived with %r" % newpacket)
self.ackedCount += 1
packetID = global_id.CanonicalID(newpacket.PacketID)
if packetID not in self.fileSendQueue:
lg.warn("packet %s not in sending queue for %s" %
(newpacket.PacketID, self.remoteName))
return
if packetID not in list(self.fileSendDict.keys()):
lg.warn("packet %s not in sending dict for %s" %
(newpacket.PacketID, self.remoteName))
return
f_up = self.fileSendDict[packetID]
if newpacket.Command == commands.Ack():
f_up.event('ack-received', newpacket)
elif newpacket.Command == commands.Fail():
f_up.event('fail-received', newpacket)
else:
raise Exception('wrong command received in response: %r' %
newpacket)
from customer import supplier_connector
sc = supplier_connector.by_idurl(newpacket.OwnerID)
if sc:
if newpacket.Command == commands.Ack():
sc.automat('ack', newpacket)
elif newpacket.Command == commands.Fail():
sc.automat('fail', newpacket)
else:
raise Exception('incorrect packet type received: %r' %
newpacket)
else:
lg.warn('supplier connector for %r not found' % newpacket.OwnerID)
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.OnFileSendAckReceived %s from %s, queue=%d" %
(str(newpacket), self.remoteName, len(self.fileSendQueue)))
def doRequestService(self, *args, **kwargs):
"""
Action method.
"""
service_info = {
'needed_bytes': self.needed_bytes,
'customer_id': global_id.UrlToGlobalID(self.customer_idurl),
}
my_customer_key_id = my_id.getGlobalID(key_alias='customer')
if my_keys.is_key_registered(my_customer_key_id):
service_info['customer_public_key'] = my_keys.get_key_info(
key_id=my_customer_key_id,
include_private=False,
)
if self.key_id:
service_info['key_id'] = self.key_id
self._last_known_ecc_map = kwargs.get('ecc_map')
if self._last_known_ecc_map is not None:
service_info['ecc_map'] = self._last_known_ecc_map
self._last_known_family_position = kwargs.get('family_position')
if self._last_known_family_position is not None:
service_info['position'] = self._last_known_family_position
self._last_known_family_snapshot = kwargs.get('family_snapshot')
if self._last_known_family_snapshot is not None:
service_info['family_snapshot'] = self._last_known_family_snapshot
request = p2p_service.SendRequestService(
remote_idurl=self.supplier_idurl,
service_name='service_supplier',
json_payload=service_info,
callbacks={
commands.Ack(): self._supplier_acked,
commands.Fail(): self._supplier_failed,
},
)
self.request_packet_id = request.PacketID
def OnDataReceived(self, newpacket, result):
# if result == 'timeout':
# packetID = global_id.CanonicalID(newpacket)
# if packetID in self.fileRequestDict:
# self.fileRequestDict[packetID].fileReceivedTime = time.time()
# self.fileRequestDict[packetID].result = 'timeout'
# for callBack in self.fileRequestDict[packetID].callOnReceived:
# callBack(None, 'timeout')
# return
# we requested some data from a supplier, just received it
packetID = global_id.CanonicalID(newpacket.PacketID)
if self.shutdown:
# if we're closing down this queue (supplier replaced, don't any anything new)
if packetID in self.fileRequestDict:
for callBack in self.fileRequestDict[packetID].callOnReceived:
callBack(newpacket, 'shutdown')
if packetID in self.fileRequestDict:
del self.fileRequestDict[packetID]
lg.warn('supplier queue is shutting down')
return
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.OnDataReceived %s with result=[%s]" % (
newpacket,
result,
))
if packetID in self.fileRequestQueue:
self.fileRequestQueue.remove(packetID)
if _Debug:
lg.out(
_DebugLevel,
" removed %s from %s receiving queue, %d more items" %
(packetID, self.remoteName, len(self.fileRequestQueue)))
if newpacket.Command == commands.Data():
wrapped_packet = signed.Unserialize(newpacket.Payload)
if not wrapped_packet or not wrapped_packet.Valid():
lg.err('incoming Data() is not valid')
return
if packetID in self.fileRequestDict:
self.fileRequestDict[packetID].fileReceivedTime = time.time()
self.fileRequestDict[packetID].result = 'received'
for callBack in self.fileRequestDict[packetID].callOnReceived:
callBack(wrapped_packet, 'received')
elif newpacket.Command == commands.Fail():
if packetID in self.fileRequestDict:
self.fileRequestDict[packetID].fileReceivedTime = time.time()
self.fileRequestDict[packetID].result = 'failed'
for callBack in self.fileRequestDict[packetID].callOnReceived:
callBack(newpacket, 'failed')
else:
lg.err('incorrect response command')
if packetID in self.fileRequestDict:
del self.fileRequestDict[packetID]
if _Debug:
lg.out(
_DebugLevel,
"io_throttle.OnDataReceived %s from %s, queue=%d" %
(newpacket, self.remoteName, len(self.fileRequestQueue)))
self.DoRequest()
def doCancelService(self, *args, **kwargs):
"""
Action method.
"""
service_info = {}
my_customer_key_id = my_id.getGlobalID(key_alias='customer')
if my_keys.is_key_registered(my_customer_key_id):
service_info['customer_public_key'] = my_keys.get_key_info(
key_id=my_customer_key_id,
include_private=False,
include_signature=False,
generate_signature=False,
)
ecc_map = kwargs.get('ecc_map')
if ecc_map:
service_info['ecc_map'] = ecc_map
request = p2p_service.SendCancelService(
remote_idurl=self.supplier_idurl,
service_name='service_supplier',
json_payload=service_info,
callbacks={
commands.Ack(): self._supplier_service_acked,
commands.Fail(): self._supplier_service_failed,
},
)
self.request_packet_id = request.PacketID
def _on_my_list_files_refreshed(self, evt):
from access import shared_access_coordinator
from customer import supplier_connector
from p2p import p2p_service
from p2p import commands
for key_id in shared_access_coordinator.list_active_shares():
cur_share = shared_access_coordinator.get_active_share(key_id)
if not cur_share:
continue
if cur_share.state != 'CONNECTED':
continue
for supplier_idurl in cur_share.known_suppliers_list:
sc = supplier_connector.by_idurl(
supplier_idurl,
customer_idurl=cur_share.customer_idurl,
)
if sc is not None and sc.state == 'CONNECTED':
p2p_service.SendListFiles(
target_supplier=supplier_idurl,
customer_idurl=cur_share.customer_idurl,
key_id=cur_share.key_id,
callbacks={
commands.Files():
lambda r, i: self._on_list_files_response(
r, i, cur_share.customer_idurl, supplier_idurl,
cur_share.key_id),
commands.Fail():
lambda r, i: self._on_list_files_failed(
r, i, cur_share.customer_idurl, supplier_idurl,
cur_share.key_id),
})
def audit_private_key(key_id, untrusted_idurl, timeout=10):
"""
Be sure remote user posses given private key.
I need to posses the public key to be able to audit.
I will generate a random string, encrypt it with given key public key and send encrypted string to him.
He will decrypt and send me back original string.
Returns Deferred object.
"""
if _Debug:
lg.out(_DebugLevel, 'key_ring.audit_private_key testing %s from %s' % (key_id, untrusted_idurl))
result = Deferred()
recipient_id_obj = identitycache.FromCache(untrusted_idurl)
if not recipient_id_obj:
lg.warn('not found "%s" in identity cache' % untrusted_idurl)
result.errback(Exception('not found "%s" in identity cache' % untrusted_idurl))
return result
key_alias, creator_idurl = my_keys.split_key_id(key_id)
if not key_alias or not creator_idurl:
lg.warn('wrong key_id')
result.errback(Exception('wrong key_id'))
return result
private_test_sample = key.NewSessionKey()
if untrusted_idurl == creator_idurl and key_alias == 'master':
lg.warn('doing audit of master key (private part) of remote user')
private_test_encrypted_sample = recipient_id_obj.encrypt(private_test_sample)
else:
if not my_keys.is_key_registered(key_id):
lg.warn('unknown key: "%s"' % key_id)
result.errback(Exception('unknown key: "%s"' % key_id))
return result
private_test_encrypted_sample = my_keys.encrypt(key_id, private_test_sample)
json_payload = {
'key_id': key_id,
'audit': {
'public_sample': '',
'private_sample': base64.b64encode(private_test_encrypted_sample),
}
}
raw_payload = serialization.DictToBytes(json_payload, values_to_text=True)
block = encrypted.Block(
BackupID=key_id,
Data=raw_payload,
SessionKey=key.NewSessionKey(),
# encrypt data using public key of recipient
EncryptKey=lambda inp: recipient_id_obj.encrypt(inp),
)
encrypted_payload = block.Serialize()
p2p_service.SendAuditKey(
remote_idurl=recipient_id_obj.getIDURL(),
encrypted_payload=encrypted_payload,
packet_id=key_id,
timeout=timeout,
callbacks={
commands.Ack(): lambda response, info:
_on_audit_private_key_response(response, info, key_id, untrusted_idurl, private_test_sample, result),
commands.Fail(): lambda response, info: result.errback(Exception(response)),
None: lambda pkt_out: result.errback(Exception('timeout')), # timeout
},
)
return result
def _do_send_request_service(self, *args, **kwargs):
if len(self.request_service_packet_id) >= 3:
if _Debug:
lg.warn('too many service requests to %r' % self.router_idurl)
self.automat('service-refused', *args, **kwargs)
return
orig_identity = config.conf().getData('services/proxy-transport/my-original-identity').strip()
if not orig_identity:
orig_identity = my_id.getLocalIdentity().serialize()
service_info = {
'name': 'service_proxy_server',
'payload': {
'identity': strng.to_text(orig_identity),
},
}
service_info_raw = json.dumps(service_info)
newpacket = signed.Packet(
commands.RequestService(),
my_id.getLocalID(),
my_id.getLocalID(),
packetid.UniqueID(),
service_info_raw,
self.router_idurl,
)
packet_out.create(newpacket, wide=False, callbacks={
commands.Ack(): self._on_request_service_ack,
commands.Fail(): self._on_request_service_fail,
},)
self.request_service_packet_id.append(newpacket.PacketID)
def _do_send_identity_to_router(self, identity_source, failed_event):
try:
identity_obj = identity.identity(xmlsrc=identity_source)
except:
lg.exc()
return
if _Debug:
lg.out(_DebugLevel, 'proxy_receiver.doSendMyIdentity to %s' % self.router_idurl)
lg.out(_DebugLevel, ' contacts=%s, sources=%s' % (identity_obj.contacts, identity_obj.sources))
newpacket = signed.Packet(
commands.Identity(),
my_id.getLocalID(),
my_id.getLocalID(),
commands.Identity(),
identity_source,
self.router_idurl,
)
packet_out.create(
newpacket,
wide=True,
callbacks={
commands.Ack(): lambda response, info: self.automat('ack-received', (response, info)),
commands.Fail(): lambda x: self.automat(failed_event),
},
keep_alive=True,
)
def SendToID(idurl, Payload=None, wide=False, ack_handler=None, timeout_handler=None, response_timeout=20, ):
"""
Create ``packet`` with my Identity file and calls
``transport.gateway.outbox()`` to send it.
"""
global _PropagateCounter
if _Debug:
lg.out(_DebugLevel, "propagate.SendToID [%s] wide=%s" % (nameurl.GetName(idurl), str(wide)))
if ack_handler is None:
ack_handler = HandleAck
if timeout_handler is None:
timeout_handler = HandleTimeOut
thePayload = Payload
if thePayload is None:
thePayload = strng.to_bin(my_id.getLocalIdentity().serialize())
p = signed.Packet(
Command=commands.Identity(),
OwnerID=my_id.getIDURL(),
CreatorID=my_id.getIDURL(),
PacketID=('propagate:%d:%s' % (_PropagateCounter, packetid.UniqueID())),
Payload=thePayload,
RemoteID=idurl,
)
_PropagateCounter += 1
result = gateway.outbox(p, wide, response_timeout=response_timeout, callbacks={
commands.Ack(): ack_handler,
commands.Fail(): ack_handler,
None: timeout_handler,
})
if wide:
# this is a ping packet - need to clear old info
p2p_stats.ErasePeerProtosStates(idurl)
p2p_stats.EraseMyProtosStates(idurl)
return result
def _on_transfer_key_response(response, info, key_id, result):
if not response or not info:
if not result.called:
result.errback(Exception('timeout'))
if _Debug:
lg.warn('transfer failed, response timeout')
return None
if response.Command == commands.Ack():
if not result.called:
result.callback(response)
if _Debug:
lg.info('key %s transfer success to %s' % (key_id, response.OwnerID))
return None
if response.Command == commands.Fail():
err_msg = strng.to_text(response.Payload, errors='ignore')
if err_msg.count('key already registered'):
# it is okay to have "Fail()" response in that case
if not result.called:
result.callback(response)
if _Debug:
lg.warn('key %s already registered on %s' % (key_id, response.OwnerID))
return None
if not result.called:
result.errback(Exception(response.Payload))
if _Debug:
lg.warn('key transfer failed: %s' % response.Payload)
return None
def SendToID(idurl, ack_handler=None, Payload=None, NeedAck=False, wide=False):
"""
Create ``packet`` with my Identity file and calls
``transport.gateway.outbox()`` to send it.
"""
lg.out(
8, "propagate.SendToID [%s] wide=%s" %
(nameurl.GetName(idurl), str(wide)))
if ack_handler is None:
ack_handler = HandleAck
thePayload = Payload
if thePayload is None:
thePayload = my_id.getLocalIdentity().serialize()
p = signed.Packet(
commands.Identity(),
my_id.getLocalID(), # MyID,
my_id.getLocalID(), # MyID,
'Identity', # my_id.getLocalID(), #PacketID,
thePayload,
idurl)
# callback.register_interest(AckHandler, p.RemoteID, p.PacketID)
gateway.outbox(p,
wide,
callbacks={
commands.Ack(): ack_handler,
commands.Fail(): ack_handler,
})
if wide:
# this is a ping packet - need to clear old info
stats.ErasePeerProtosStates(idurl)
stats.EraseMyProtosStates(idurl)
def doCancelServiceQueue(self, *args, **kwargs):
"""
Action method.
"""
service_info = {
'items': [{
'scope': 'consumer',
'action': 'unsubscribe',
'consumer_id': strng.to_text(my_id.getGlobalID()),
'queue_id': global_id.MakeGlobalQueueID(
queue_alias='supplier-file-modified',
owner_id=my_id.getGlobalID(),
supplier_id=global_id.MakeGlobalID(idurl=self.supplier_idurl),
),
}, {
'scope': 'consumer',
'action': 'remove_callback',
'consumer_id': strng.to_text(my_id.getGlobalID()),
'method': strng.to_text(my_id.getLocalID()),
}, {
'scope': 'consumer',
'action': 'stop',
'consumer_id': strng.to_text(my_id.getGlobalID()),
}, ],
}
p2p_service.SendCancelService(
remote_idurl=self.supplier_idurl,
service_name='service_p2p_notifications',
json_payload=service_info,
callbacks={
commands.Ack(): self._supplier_acked,
commands.Fail(): self._supplier_failed,
},
)
def _do_retrieve(self, x=None):
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
localID = my_id.getIDURL()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
pkt_out = p2p_service.SendRetreive(ownerID=localID,
creatorID=localID,
packetID=packetID,
remoteID=supplier_idurl,
response_timeout=60 * 2,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_fail,
})
if pkt_out:
self.requesting_suppliers.add(supplier_idurl)
self.requested_suppliers_number += 1
self.requests_packets_sent.append((packetID, supplier_idurl))
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplier_idurl)))
def DataReceived(self, newpacket, info):
# we requested some data from a supplier, just received it
if self.shutdown:
# if we're closing down this queue (supplier replaced, don't any anything new)
return
packetID = global_id.CanonicalID(newpacket.PacketID)
if packetID in self.fileRequestQueue:
self.fileRequestQueue.remove(packetID)
if newpacket.Command == commands.Data():
if packetID in self.fileRequestDict:
self.fileRequestDict[packetID].fileReceivedTime = time.time()
self.fileRequestDict[packetID].result = 'received'
for callBack in self.fileRequestDict[packetID].callOnReceived:
callBack(newpacket, 'received')
elif newpacket.Command == commands.Fail():
if packetID in self.fileRequestDict:
self.fileRequestDict[packetID].fileReceivedTime = time.time()
self.fileRequestDict[packetID].result = 'failed'
for callBack in self.fileRequestDict[packetID].callOnReceived:
callBack(newpacket, 'failed')
else:
raise Exception('incorrect response command')
if packetID in self.fileRequestDict:
del self.fileRequestDict[packetID]
lg.out(
10, "io_throttle.DataReceived %s from %s, queue=%d" %
(newpacket, self.remoteName, len(self.fileRequestQueue)))
self.DoRequest()
def _on_inbox_packet_received(self, newpacket, info, status,
error_message):
if newpacket.Command == commands.Identity() and \
newpacket.CreatorID == self.router_idurl and \
newpacket.RemoteID == my_id.getLocalID():
self.automat('router-id-received', (newpacket, info))
self.latest_packet_received = time.time()
return True
if newpacket.Command == commands.Fail() and \
newpacket.CreatorID == self.router_idurl and \
newpacket.RemoteID == my_id.getLocalID():
self.automat('service-refused', (newpacket, info))
return True
if newpacket.CreatorID == self.router_idurl:
self.latest_packet_received = time.time()
if newpacket.Command != commands.Relay():
return False
# if not newpacket.PacketID.startswith('routed_in_'):
# return False
# if newpacket.RemoteID != my_id.getLocalID():
# return False
# if newpacket.CreatorID != self.router_idurl:
# return False
self.automat('inbox-packet', (newpacket, info, status, error_message))
return True
def doSendRequestService(self, arg):
"""
Action method.
"""
if len(self.request_service_packet_id) >= 3:
if _Debug:
lg.warn('too many service requests to %s' % self.router_idurl)
self.automat('service-refused', arg)
return
service_info = 'service_proxy_server \n'
orig_identity = config.conf().getData(
'services/proxy-transport/my-original-identity').strip()
if not orig_identity:
orig_identity = my_id.getLocalIdentity().serialize()
service_info += orig_identity
# for t in gateway.transports().values():
# service_info += '%s://%s' % (t.proto, t.host)
# service_info += ' '
newpacket = signed.Packet(
commands.RequestService(),
my_id.getLocalID(),
my_id.getLocalID(),
packetid.UniqueID(),
service_info,
self.router_idurl,
)
packet_out.create(
newpacket,
wide=False,
callbacks={
commands.Ack(): self._on_request_service_ack,
commands.Fail(): self._on_request_service_fail
},
)
self.request_service_packet_id.append(newpacket.PacketID)
def isAckNeeded(self, *args, **kwargs):
"""
Condition method.
"""
return commands.Ack() in list(
self.callbacks.keys()) or commands.Fail() in list(
self.callbacks.keys())
def _do_request_list_files(self, suppliers_list):
backup_matrix.add_list_files_query_callback(
customer_idurl=self.queue_owner_idurl,
query_path=self.queue_alias,
callback_method=self._on_list_files_response,
)
self.correctable_errors = eccmap.GetCorrectableErrors(
len(suppliers_list))
for supplier_pos, supplier_idurl in enumerate(suppliers_list):
if not supplier_idurl:
self.requested_list_files[supplier_pos] = False
continue
outpacket = p2p_service.SendListFiles(
target_supplier=supplier_idurl,
key_id=self.group_key_id,
query_items=[
self.queue_alias,
],
timeout=30,
callbacks={
commands.Fail():
lambda resp, info: self._on_list_files_failed(supplier_pos
),
None:
lambda pkt_out: self._on_list_files_failed(supplier_pos),
},
)
self.requested_list_files[
supplier_pos] = None if outpacket else False
if _Debug:
lg.args(_DebugLevel, requested=self.requested_list_files)
self.request_list_files_timer = reactor.callLater(
30, self._on_request_list_files_timeout) # @UndefinedVariable
def doSendMyListFiles(self, *args, **kwargs):
"""
Action method.
"""
json_list_files = backup_fs.Serialize(
to_json=True,
filter_cb=lambda path_id, path, info: True if strng.to_text(info.key_id) == strng.to_text(self.key_id) else False,
)
# raw_list_files = json.dumps(json_list_files, indent=2, encoding='utf-8')
raw_list_files = serialization.DictToBytes(json_list_files, keys_to_text=True, values_to_text=True)
if _Debug:
lg.out(_DebugLevel, 'shared_access_donor.doSendMyListFiles prepared list of files for %s :\n%s' % (
self.remote_idurl, raw_list_files))
block = encrypted.Block(
CreatorID=my_id.getLocalID(),
BackupID=self.key_id,
Data=raw_list_files,
SessionKey=key.NewSessionKey(),
EncryptKey=self.key_id,
)
encrypted_list_files = block.Serialize()
packet_id = "%s:%s" % (self.key_id, packetid.UniqueID(), )
p2p_service.SendFiles(
idurl=self.remote_idurl,
raw_list_files_info=encrypted_list_files,
packet_id=packet_id,
callbacks={
commands.Ack(): lambda response, _: self.automat('list-files-ok', response),
commands.Fail(): lambda response, _: self.automat('fail', Exception(str(response))),
None: lambda pkt_out: self.automat('fail', Exception('timeout')),
},
)
def handle(newpacket, info):
from transport import packet_out
handled = False
# check that signed by a contact of ours
if not newpacket.Valid():
lg.warn('new packet from %s://%s is NOT VALID: %r' % (
info.proto, info.host, newpacket))
return None
for p in packet_out.search_by_response_packet(newpacket, info.proto, info.host):
p.automat('inbox-packet', (newpacket, info))
handled = True
if _Debug:
lg.out(_DebugLevel, ' processed by %s as response packet' % p)
handled = callback.run_inbox_callbacks(newpacket, info, info.status, info.error_message) or handled
if not handled and newpacket.Command not in [commands.Ack(), commands.Fail(), commands.Identity(), ]:
lg.warn('incoming %s from [%s://%s] was NOT HANDLED' % (newpacket, info.proto, info.host))
if _Debug:
history().append({
'time': newpacket.Date,
'command': newpacket.Command,
'packet_id': newpacket.PacketID,
'creator_id': newpacket.CreatorID,
'owner_id': newpacket.OwnerID,
'remote_id': newpacket.RemoteID,
'payload': len(newpacket.Payload),
'address': '%s://%s' % (info.proto, info.host),
})
if len(history()) > 100:
history().pop(0)
return handled
def doRequestService(self, arg):
"""
Action method.
"""
service_info = {
'needed_bytes': self.needed_bytes,
'customer_id': global_id.UrlToGlobalID(self.customer_idurl),
}
my_customer_key_id = my_id.getGlobalID(key_alias='customer')
if my_keys.is_key_registered(my_customer_key_id):
service_info['customer_public_key'] = my_keys.get_key_info(
key_id=my_customer_key_id,
include_private=False,
)
if self.key_id:
service_info['key_id'] = self.key_id
if self.customer_idurl == my_id.getLocalIDURL():
service_info['ecc_map'] = eccmap.Current().name
request = p2p_service.SendRequestService(
remote_idurl=self.supplier_idurl,
service_name='service_supplier',
json_payload=service_info,
callbacks={
commands.Ack(): self._supplier_acked,
commands.Fail(): self._supplier_failed,
},
)
self.request_packet_id = request.PacketID
def doSendRequestService(self, *args, **kwargs):
"""
Action method.
"""
self.target_idurl.refresh()
packet_id = packetid.UniqueID()
if _Debug:
lg.args(_DebugLevel,
idurl=self.target_idurl,
service=self.target_service,
packet_id=packet_id)
service_request_payload = self.request_service_params
if callable(service_request_payload):
service_request_payload = service_request_payload(
self.target_idurl)
out_packet = p2p_service.SendRequestService(
remote_idurl=self.target_idurl,
service_name=self.target_service,
json_payload=service_request_payload,
timeout=self.request_service_timeout,
callbacks={
commands.Ack(): self._node_acked,
commands.Fail(): self._node_failed,
None: self._node_timed_out,
},
packet_id=packet_id,
)
self.requested_packet_id = out_packet.PacketID
def OnDataReceived(self, newpacket, result):
# we requested some data from a supplier, and just received it
if self.shutdown:
lg.warn('skip, supplier queue is shutting down')
self.StopAllRequests()
return
if _Debug:
lg.args(_DebugLevel, newpacket=newpacket, result=result, queue=len(self.fileRequestQueue), remoteName=self.remoteName)
packetID = global_id.CanonicalID(newpacket.PacketID)
if (packetID not in self.fileRequestQueue) or (packetID not in self.fileRequestDict):
latest_idurl = global_id.NormalizeGlobalID(packetID, as_field=True)['idurl'].latest
another_packetID = global_id.SubstitutePacketID(packetID, idurl=latest_idurl)
if (another_packetID in self.fileRequestQueue) and (another_packetID in self.fileRequestDict):
packetID = another_packetID
lg.warn('found incoming %r with outdated packet id, corrected: %r' % (newpacket, another_packetID, ))
if (packetID not in self.fileRequestQueue) or (packetID not in self.fileRequestDict):
lg.err('unexpected %r received which is not in the downloading queue' % newpacket)
else:
f_down = self.fileRequestDict[packetID]
if newpacket.Command == commands.Data():
wrapped_packet = signed.Unserialize(newpacket.Payload)
if not wrapped_packet or not wrapped_packet.Valid():
lg.err('incoming Data() packet is not valid')
f_down.event('fail-received', newpacket)
return
f_down.event('valid-data-received', wrapped_packet)
elif newpacket.Command == commands.Fail():
f_down.event('fail-received', newpacket)
else:
lg.err('incorrect response command: %r' % newpacket)
def _do_send_request_service(self, *args, **kwargs):
if len(self.request_service_packet_id) >= 10:
if _Debug:
lg.warn('too many service requests to %r' % self.router_idurl)
self.automat('service-refused', *args, **kwargs)
return
orig_identity = config.conf().getData('services/proxy-transport/my-original-identity').strip()
if not orig_identity:
orig_identity = my_id.getLocalIdentity().serialize(as_text=True)
service_info = {
'name': 'service_proxy_server',
'payload': {
'identity': orig_identity,
},
}
newpacket = signed.Packet(
commands.RequestService(),
my_id.getIDURL(),
my_id.getIDURL(),
packetid.UniqueID(),
serialization.DictToBytes(service_info, values_to_text=True),
self.router_idurl,
)
packet_out.create(
newpacket,
wide=False,
callbacks={
commands.Ack(): self._on_request_service_ack,
commands.Fail(): self._on_request_service_fail,
None: lambda pkt_out: self.automat('request-timeout', pkt_out),
},
response_timeout=30,
)
self.request_service_packet_id.append(newpacket.PacketID)
def _do_send_identity_to_router(self, identity_source, failed_event):
try:
identity_obj = identity.identity(xmlsrc=identity_source)
except:
lg.exc()
return
if _Debug:
lg.out(_DebugLevel, 'proxy_receiver._do_send_identity_to_router to %s' % self.router_idurl)
lg.out(_DebugLevel, ' contacts=%r, sources=%r' % (
identity_obj.contacts, identity_obj.getSources(as_originals=True)))
newpacket = signed.Packet(
Command=commands.Identity(),
OwnerID=my_id.getIDURL(),
CreatorID=my_id.getIDURL(),
PacketID=('proxy_receiver:%s' % packetid.UniqueID()),
Payload=identity_obj.serialize(),
RemoteID=self.router_idurl,
)
packet_out.create(
newpacket,
wide=True,
callbacks={
commands.Ack(): lambda response, info: self.automat('ack-received', (response, info)),
commands.Fail(): lambda x: self.automat(failed_event),
None: lambda pkt_out: self.automat('ack-timeout', pkt_out),
'failed': lambda pkt_out, error_message: self.automat('sending-failed', (pkt_out, error_message)),
},
keep_alive=True,
response_timeout=30,
)
def OnDataReceived(self, newpacket, result):
# we requested some data from a supplier, and just received it
if self.shutdown:
lg.warn('supplier queue is shutting down')
self.StopAllRequests()
return
if _Debug:
lg.args(_DebugLevel,
newpacket=newpacket,
result=result,
queue=len(self.fileRequestQueue),
remoteName=self.remoteName)
packetID = global_id.CanonicalID(newpacket.PacketID)
if (packetID
not in self.fileRequestQueue) or (packetID
not in self.fileRequestDict):
lg.err(
'unexpected %r received which is not in the downloading queue'
% newpacket)
else:
f_down = self.fileRequestDict[packetID]
if newpacket.Command == commands.Data():
wrapped_packet = signed.Unserialize(newpacket.Payload)
if not wrapped_packet or not wrapped_packet.Valid():
lg.err('incoming Data() packet is not valid')
f_down.event('fail-received', newpacket)
return
f_down.event('valid-data-received', wrapped_packet)
elif newpacket.Command == commands.Fail():
f_down.event('fail-received', newpacket)
else:
lg.err('incorrect response command: %r' % newpacket)
def SendToIDs(idlist, wide=False, ack_handler=None, timeout_handler=None, response_timeout=20):
"""
Same, but send to many IDs and also check previous packets to not re-send.
"""
global _PropagateCounter
if _Debug:
lg.out(_DebugLevel, "propagate.SendToIDs to %d users, wide=%s" % (len(idlist), wide))
if ack_handler is None:
ack_handler = HandleAck
if timeout_handler is None:
timeout_handler = HandleTimeOut
LocalIdentity = my_id.getLocalIdentity()
Payload = strng.to_bin(LocalIdentity.serialize())
alreadysent = set()
totalsent = 0
inqueue = {}
found_previous_packets = 0
for pkt_out in packet_out.queue():
if id_url.is_in(pkt_out.remote_idurl, idlist, as_field=False):
if pkt_out.description.count('Identity'):
if pkt_out.remote_idurl not in inqueue:
inqueue[pkt_out.remote_idurl] = 0
inqueue[pkt_out.remote_idurl] += 1
found_previous_packets += 1
for contact in idlist:
if not contact:
continue
if contact in alreadysent:
# just want to send once even if both customer and supplier
continue
if contact in inqueue and inqueue[contact] > 2:
# now only 2 protocols is working: tcp and udp
if _Debug:
lg.out(_DebugLevel, ' skip sending [Identity] to %s, packet already in the queue' % contact)
continue
p = signed.Packet(
Command=commands.Identity(),
OwnerID=my_id.getLocalID(),
CreatorID=my_id.getLocalID(),
PacketID=('propagate:%d:%s' % (_PropagateCounter, packetid.UniqueID())),
Payload=Payload,
RemoteID=contact,
)
_PropagateCounter += 1
if _Debug:
lg.out(_DebugLevel, " sending [Identity] to %s" % nameurl.GetName(contact))
gateway.outbox(p, wide, response_timeout=response_timeout, callbacks={
commands.Ack(): ack_handler,
commands.Fail(): ack_handler,
None: timeout_handler,
})
if wide:
# this is a ping packet - need to clear old info
p2p_stats.ErasePeerProtosStates(contact)
p2p_stats.EraseMyProtosStates(contact)
alreadysent.add(contact)
totalsent += 1
del alreadysent
return totalsent
def do_notify(callback_method, consumer_id, queue_id, message_id):
existing_message = queue(queue_id)[message_id]
event_id = global_id.ParseGlobalQueueID(queue_id)['queue_alias']
if consumer_id in existing_message.notifications:
if _Debug:
lg.dbg(
_DebugLevel,
'notification %r already started for consumer %r' % (
message_id,
consumer_id,
))
# notification already sent to given consumer
return False
ret = Deferred()
start_notification(consumer_id, queue_id, message_id, ret)
if id_url.is_idurl(callback_method):
p2p_service.SendEvent(
remote_idurl=id_url.field(callback_method),
event_id=event_id,
payload=existing_message.payload,
producer_id=existing_message.producer_id,
consumer_id=consumer_id,
queue_id=queue_id,
message_id=existing_message.message_id,
created=existing_message.created,
response_timeout=15,
callbacks={
commands.Ack(): lambda response, info: ret.callback(True),
commands.Fail(): lambda response, info: ret.callback(False),
None: lambda pkt_out: ret.callback(False),
},
)
else:
try:
result = callback_method(
dict(
event_id=event_id,
payload=existing_message.payload,
producer_id=existing_message.producer_id,
consumer_id=consumer_id,
queue_id=queue_id,
message_id=existing_message.message_id,
created=existing_message.created,
))
except:
lg.exc()
result = False
if isinstance(result, Deferred):
result.addCallback(lambda ok: ret.callback(True)
if ok else ret.callback(False))
if _Debug:
result.addErrback(lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='p2p_queue.do_notify')
result.addErrback(lambda err: ret.callback(False))
else:
reactor.callLater(0, ret.callback, result) # @UndefinedVariable
return ret