def doVerifyDHTRecords(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.args(_DebugLevel,
my=self.my_position,
desired=self.desired_position,
dht=self.dht_brokers)
if not self.dht_brokers.get(self.desired_position):
# desired position is empty in DHT
if self.desired_position == 0:
self.automat('top-record-empty')
else:
prev_position = self.desired_position - 1
if not self.dht_brokers.get(prev_position):
# also previous position in DHT is empty as well
self.automat('prev-record-empty')
else:
if id_url.is_the_same(self.dht_brokers[prev_position],
self.my_broker_idurl):
# found my DHT record on a previous position - request was done on another position
self.automat('prev-record-own',
dht_brokers=self.dht_brokers)
else:
# found another broker on the previous position in DHT
self.automat('prev-record-busy')
return
if id_url.is_the_same(self.dht_brokers[self.desired_position],
self.my_broker_idurl):
# my own record is present in DHT on that position
if self.desired_position == 0:
self.automat('top-record-own')
else:
prev_position = self.desired_position - 1
if not self.dht_brokers.get(prev_position):
# but the previous position in DHT is empty
self.automat('prev-record-empty')
else:
if id_url.is_the_same(self.dht_brokers[prev_position],
self.my_broker_idurl):
# found my DHT record on a previous position - this is wrong!
self.automat('prev-record-own',
dht_brokers=self.dht_brokers)
else:
# found another broker on the previous position in DHT
self.automat('prev-record-busy')
# self.automat('record-own')
return
# desired position is occupied by another broker in DHT records
if self.desired_position == 0:
self.automat('top-record-busy', dht_brokers=self.dht_brokers)
else:
self.automat('record-busy')
def _on_rotate_broker_connected(self, response_info, broker_pos, event,
*args, **kwargs):
try:
# skip leading "accepted:" marker
cooperated_brokers = jsn.loads(
strng.to_text(response_info[0].Payload)[9:])
cooperated_brokers.pop('archive_folder_path', None)
cooperated_brokers = {
int(k): id_url.field(v)
for k, v in cooperated_brokers.items()
}
except:
lg.exc()
self.automat('broker-rotate-failed')
return
if _Debug:
lg.args(_DebugLevel,
cooperated=cooperated_brokers,
pos=broker_pos,
e=event)
if id_url.is_the_same(cooperated_brokers.get(broker_pos),
self.my_broker_idurl):
self.automat('broker-rotate-accepted',
cooperated_brokers=cooperated_brokers)
return
self.automat('broker-rotate-rejected',
cooperated_brokers=cooperated_brokers)
def _do_dht_push_state(self, event=None, **kwargs):
if _Debug:
lg.args(_DebugLevel, known_position=self.known_position, cooperated_brokers=self.cooperated_brokers, e=event, kw=kwargs)
desired_position = self.known_position
if desired_position is None or desired_position == -1:
for pos, idurl in self.cooperated_brokers.items():
if idurl and id_url.is_the_same(idurl, self.broker_idurl):
desired_position = pos
if desired_position is None or desired_position == -1:
raise Exception('not able to write record into DHT, my position is unknown')
archive_folder_path = None
if self.current_connect_request and self.current_connect_request.get('archive_folder_path'):
archive_folder_path = self.current_connect_request['archive_folder_path']
prev_revision = self.latest_dht_records.get(desired_position, {}).get('revision', None)
if prev_revision is None:
prev_revision = 0
if _Debug:
lg.args(_DebugLevel, c=self.customer_id, p=desired_position, b=self.broker_id, r=prev_revision+1)
self._do_dht_write(
desired_position=desired_position,
archive_folder_path=archive_folder_path,
revision=prev_revision+1,
retry=True,
event=event,
**kwargs,
)
def _on_new_broker_hired(self, response_info, broker_pos, my_pos,
desired_pos, *args, **kwargs):
try:
# skip leading "accepted:" marker
cooperated_brokers = jsn.loads(
strng.to_text(response_info[0].Payload)[9:])
cooperated_brokers.pop('archive_folder_path', None)
cooperated_brokers = {
int(k): id_url.field(v)
for k, v in cooperated_brokers.items()
}
except:
lg.exc()
self.automat('hire-broker-failed')
return
if _Debug:
lg.args(_DebugLevel,
cooperated=cooperated_brokers,
target=broker_pos,
my=my_pos,
desired=desired_pos,
args=args,
kwargs=kwargs)
if my_pos >= 0:
if id_url.is_the_same(cooperated_brokers.get(my_pos),
self.my_broker_idurl):
self.automat('hire-broker-ok',
cooperated_brokers=cooperated_brokers)
return
if desired_pos >= 0:
if id_url.is_the_same(cooperated_brokers.get(desired_pos),
self.my_broker_idurl):
self.automat('hire-broker-ok',
cooperated_brokers=cooperated_brokers)
return
lg.warn(
'new broker is not cooperative, my idurl is not found in the cooperation on the right place'
)
self.automat('new-broker-rejected',
cooperated_brokers=cooperated_brokers)
def verify_broker(self, broker_idurl, position, known_brokers, known_streams):
if not self.InSync or self.state != 'CONNECTED':
lg.warn('not able to verify another broker because %r is not in sync' % self)
return None
if self.cooperated_brokers:
if id_url.is_not_in(broker_idurl, self.cooperated_brokers.values()):
return 'unknown broker'
if position not in self.cooperated_brokers.keys():
return 'unknown position'
if not id_url.is_the_same(self.cooperated_brokers[position], broker_idurl):
return 'position mismatch'
if len(self.cooperated_brokers) != len(known_brokers):
return 'brokers count mismatch'
for i, b_idurl in self.cooperated_brokers.items():
if not id_url.is_the_same(known_brokers[i], b_idurl):
return 'broker mismatch'
if self.known_streams and known_streams:
for my_queue_alias, my_archive_folder_path in self.known_streams.items():
for other_queue_alias, other_archive_folder_path in self.known_streams.items():
# TODO: verify streams
pass
return None
def doRememberOwnPosition(self, *args, **kwargs):
"""
Action method.
"""
my_new_position = -1
for pos, idurl in self.cooperated_brokers.items():
if idurl and id_url.is_the_same(idurl, self.broker_idurl):
my_new_position = pos
archive_folder_path = kwargs.get('archive_folder_path')
if _Debug:
lg.args(_DebugLevel, known=self.known_position, new=my_new_position, cooperated=self.cooperated_brokers, af_path=archive_folder_path)
if my_new_position >= 0:
self.known_position = my_new_position
if self.current_connect_request and self.current_connect_request.get('request') == 'connect' and archive_folder_path:
try:
self.known_streams[self.current_connect_request['group_key_info']['alias']] = archive_folder_path
except:
lg.exc(str(self.current_connect_request))
def isAnyFilesShared(self, *args, **kwargs):
"""
Condition method.
"""
if id_url.is_the_same(self.customer_idurl, my_id.getIDURL()):
if _Debug:
lg.args(_DebugLevel,
c=self.customer_idurl,
key_alias=self.key_alias,
ret='my own share')
# no need to wait for incoming list files
# TODO: check what is going to happen when new file is uploaded to the share by another user (not creator of the share)
return True
ret = backup_fs.HasChilds('',
iter=backup_fs.fs(self.customer_idurl,
self.key_alias))
if _Debug:
lg.args(_DebugLevel,
customer=self.customer_idurl,
key_alias=self.key_alias,
ret=ret)
return ret
def on_files_received(newpacket, info):
list_files_global_id = global_id.ParseGlobalID(newpacket.PacketID)
if not list_files_global_id['idurl']:
lg.warn('invalid PacketID: %s' % newpacket.PacketID)
return False
if not id_url.is_the_same(list_files_global_id['idurl'], my_id.getIDURL()):
# ignore Files() if this is another customer
if _Debug:
lg.dbg(_DebugLevel, 'ignore incoming %r which is owned by another customer' % newpacket)
return False
if not contactsdb.is_supplier(newpacket.OwnerID):
# ignore Files() if this is not my supplier
if _Debug:
lg.dbg(_DebugLevel, 'incoming %r received, but %r is not my supplier' % (newpacket, newpacket.OwnerID, ))
return False
if _Debug:
lg.args(_DebugLevel, "service_backups._on_inbox_packet_received: %r for us from %s at %s" % (
newpacket, newpacket.CreatorID, info))
if IncomingSupplierListFiles(newpacket, list_files_global_id):
p2p_service.SendAck(newpacket)
else:
p2p_service.SendFail(newpacket)
return True
def on_identity_url_changed(evt):
from access import group_member
service_dir = settings.ServiceDir('service_private_groups')
groups_dir = os.path.join(service_dir, 'groups')
brokers_dir = os.path.join(service_dir, 'brokers')
old_idurl = id_url.field(evt.data['old_idurl'])
new_idurl = id_url.field(evt.data['new_idurl'])
active_group_keys = list(active_groups())
to_be_reconnected = []
for group_key_id in active_group_keys:
if not group_key_id:
continue
group_creator_idurl = global_id.glob2idurl(group_key_id)
if id_url.is_the_same(group_creator_idurl, old_idurl):
old_group_path = os.path.join(groups_dir, group_key_id)
latest_group_key_id = my_keys.latest_key_id(group_key_id)
latest_group_path = os.path.join(groups_dir, latest_group_key_id)
lg.info('going to rename rotated group file: %r -> %r' % (old_group_path, latest_group_path, ))
if os.path.isfile(old_group_path):
try:
os.rename(old_group_path, latest_group_path)
except:
lg.exc()
continue
else:
lg.warn('key file %r was not found, key was not renamed' % old_group_path)
active_groups()[latest_group_key_id] = active_groups().pop(group_key_id)
group_member.rotate_active_group_memeber(group_key_id, latest_group_key_id)
gm = group_member.get_active_group_member(group_key_id)
if gm and gm.connected_brokers and id_url.is_in(old_idurl, gm.connected_brokers.values()):
lg.info('connected broker %r IDURL is rotated, going to reconnect %r' % (old_idurl, gm, ))
if group_key_id not in to_be_reconnected:
to_be_reconnected.append(group_key_id)
known_customers = list(known_brokers().keys())
for customer_id in known_customers:
latest_customer_id = global_id.idurl2glob(new_idurl)
customer_idurl = global_id.glob2idurl(customer_id)
if id_url.is_the_same(customer_idurl, old_idurl):
latest_customer_dir = os.path.join(brokers_dir, latest_customer_id)
lg.info('going to rename rotated customer id: %r -> %r' % (customer_id, latest_customer_id, ))
old_customer_dir = os.path.join(brokers_dir, customer_id)
if os.path.isdir(old_customer_dir):
try:
bpio.move_dir_recursive(old_customer_dir, latest_customer_dir)
bpio.rmdir_recursive(old_customer_dir)
except:
lg.exc()
continue
known_brokers()[latest_customer_id] = known_brokers().pop(customer_id)
for broker_pos, broker_id in enumerate(known_brokers(latest_customer_id)):
if not broker_id:
continue
broker_idurl = global_id.glob2idurl(broker_id)
if broker_idurl == old_idurl:
latest_broker_id = global_id.idurl2glob(new_idurl)
latest_broker_path = os.path.join(latest_customer_dir, latest_broker_id)
lg.info('going to rename rotated broker id: %r -> %r' % (broker_id, latest_broker_id, ))
old_broker_path = os.path.join(latest_customer_dir, broker_id)
if os.path.isfile(old_broker_path):
try:
os.rename(old_broker_path, latest_broker_path)
except:
lg.exc()
continue
if latest_broker_id in known_brokers(latest_customer_id):
lg.warn('broker %r already exist' % latest_broker_id)
continue
known_brokers()[latest_customer_id][broker_pos] = latest_broker_id
if _Debug:
lg.args(_DebugLevel, to_be_reconnected=to_be_reconnected)
for group_key_id in to_be_reconnected:
gm = group_member.get_active_group_member(group_key_id)
if gm:
gm.automat('reconnect')
def doVerifyKnownBrokers(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.args(_DebugLevel,
my=self.my_position,
desired=self.desired_position,
cooperated=self.cooperated_brokers)
# cooperation was done before with other brokers and my own position is known already to me
if self.desired_position != self.my_position:
# but the request was done to a wrong position
lg.warn(
'requester desired position %d mismatch, my current position is: %d'
% (
self.desired_position,
self.my_position,
))
self.automat(
'request-invalid',
Exception('position mismatch, current position is: %d' %
self.my_position),
cooperated_brokers=self.cooperated_brokers)
return
if not self.cooperated_brokers.get(self.my_position):
# there is no broker present in the cooperation for my position
lg.err(
'my current cooperated info is not valid, there is no broker on position %d'
% self.my_position)
self.automat(
'request-invalid',
Exception(
'current cooperation is not valid, there is no broker on position %d'
% self.my_position))
return
if not id_url.is_the_same(self.cooperated_brokers[self.my_position],
self.my_broker_idurl):
# looks like my current deal is not correct - another broker is taking my position
lg.err(
'my current cooperated info is not correct, another broker is taking my position'
)
self.automat(
'request-invalid',
Exception(
'current cooperation is not valid, another broker is taking position %d'
% self.my_position))
return
if self.my_position > 0:
prev_position = self.my_position - 1
if not self.cooperated_brokers.get(prev_position):
# but on the previous position there is no broker present in the cooperation
lg.err(
'my current cooperated info is not valid, there is no broker on previous position %d'
% prev_position)
self.automat(
'request-invalid',
Exception(
'current cooperation is not valid, there is no broker on previous position %d'
% prev_position))
return
# requester is trying to connect to me on the correct position
if self.requestor_known_brokers.get(self.my_position):
if not id_url.is_the_same(
self.requestor_known_brokers[self.my_position],
self.my_broker_idurl):
# but there is a request to change the cooperation - it looks like a trigger for a brokers rotation
lg.warn(
'received a request to change the cooperation, another broker %r going to replace me on position %d'
% (
self.requestor_known_brokers[self.my_position],
self.my_position,
))
if not self.dht_brokers.get(self.my_position):
# there is no record in DHT for my position
# my info is not stored in DHT and another broker is going to replace me
# but there is already a cooperation done before and my own position is known to me
if self.my_position == 0:
self.automat('my-top-record-empty-replace')
else:
self.automat('my-record-empty-replace')
return
# there is a record in DHT on my expected position while another broker is trying to replace me
if not id_url.is_the_same(self.dht_brokers[self.my_position],
self.my_broker_idurl):
# DHT record on my expected position is occupied by another broker
lg.warn(
'DHT record on my expected position %d is occupied by another broker %r and also another broker is trying to replace me: %r'
% (
self.my_position,
self.dht_brokers[self.my_position],
self.requestor_known_brokers[self.my_position],
))
if self.my_position == 0:
self.automat('my-top-record-busy-replace')
else:
self.automat('my-record-busy-replace')
return
# found my own record on the expected position in DHT while another broker is trying to replace me
if self.my_position == 0:
self.automat('my-top-record-own-replace')
else:
self.automat('my-record-own-replace')
return
# requester is not going to change the existing cooperation for my own position
if not self.dht_brokers.get(self.my_position):
# but in DHT my own record is missing on the expected position
lg.warn('DHT record on my expected position %d is empty' %
self.my_position)
if self.my_position == 0:
self.automat('my-top-record-empty')
else:
self.automat('my-record-empty')
return
# there is a record in DHT on my expected position
if not id_url.is_the_same(self.dht_brokers[self.my_position],
self.my_broker_idurl):
# DHT record on my expected position is occupied by another broker
lg.warn(
'DHT record on my expected position %d is occupied by another broker: %r'
% (
self.my_position,
self.dht_brokers[self.my_position],
))
if self.my_position == 0:
self.automat('my-top-record-busy')
else:
self.automat('my-record-busy')
return
# found my own record on expected position in DHT
if self.my_position == 0:
self.automat('my-top-record-own')
else:
self.automat('my-record-own')
def _on_packet_request_result(self, NewPacketOrPacketID, result):
if self.block_requests is None:
return
if _Debug:
lg.args(_DebugLevel, packet=NewPacketOrPacketID, result=result)
packet_id = None
if strng.is_string(NewPacketOrPacketID):
packet_id = NewPacketOrPacketID
else:
packet_id = getattr(NewPacketOrPacketID, 'PacketID', None)
if not packet_id:
raise Exception('packet ID is unknown from %r' %
NewPacketOrPacketID)
if packet_id not in self.block_requests:
resp = global_id.NormalizeGlobalID(packet_id)
for req_packet_id in self.block_requests:
req = global_id.NormalizeGlobalID(req_packet_id)
if resp['version'] == req['version'] and resp['path'] == req[
'path']:
if resp['key_alias'] == req['key_alias'] and resp[
'user'] == req['user']:
if id_url.is_the_same(resp['idurl'], req['idurl']):
packet_id = req_packet_id
lg.warn(
'found matching packet request %r for rotated idurl %r'
% (
packet_id,
resp['idurl'],
))
break
if packet_id not in self.block_requests:
if _Debug:
lg.args(_DebugLevel, block_requests=self.block_requests)
raise Exception('packet ID not registered')
if result == 'in queue':
if self.block_requests[packet_id] is not None:
raise Exception(
'packet is still in IO queue, but already unregistered')
lg.warn('packet already in the request queue: %r' % packet_id)
return
if result in [
'received',
'exist',
]:
self.block_requests[packet_id] = True
if result == 'exist':
# reactor.callLater(0, self.automat, 'data-received', (None, packet_id, )) # @UndefinedVariable
self.event('data-received', (
None,
packet_id,
))
else:
# reactor.callLater(0, self.automat, 'data-received', (NewPacketOrPacketID, packet_id, )) # @UndefinedVariable
self.event('data-received', (
NewPacketOrPacketID,
packet_id,
))
else:
self.block_requests[packet_id] = False
self.RequestFails.append(packet_id)
# reactor.callLater(0, self.automat, 'request-failed', packet_id) # @UndefinedVariable
self.event('request-failed', packet_id)
def contacts_remote(include_all=False, include_enabled=True):
"""
Return ID's list of all known peers.
"""
l = id_url.to_bin_list(contacts(include_all=include_all, include_enabled=include_enabled))
return [i for i in id_url.fields_list(l) if not id_url.is_the_same(i, my_id.getIDURL())]
