def test_two_sources(self):
with self.assertRaises(KeyError):
{
id_url.field(frank_1): 'frank1',
}
with self.assertRaises(KeyError):
(id_url.field(frank_1) in {})
with self.assertRaises(KeyError):
(id_url.field(frank_2) in {})
self._cache_identity('frank')
d1 = {
id_url.field(frank_1): 'frank1',
}
d2 = {
id_url.field(frank_2): 'frank2',
}
self.assertEqual(id_url.field(frank_1), id_url.field(frank_2))
self.assertIn(id_url.field(frank_1), d1)
self.assertIn(id_url.field(frank_2), d2)
self.assertIn(id_url.field(frank_1), d2)
self.assertIn(id_url.field(frank_2), d1)
self.assertTrue(id_url.is_in(frank_1, d2))
self.assertTrue(id_url.is_in(frank_2, d2))
self.assertEqual(list(d1.keys())[0], list(d2.keys())[0])
self.assertEqual(list(d1.values())[0], 'frank1')
self.assertEqual(d1.keys(), d2.keys())
self.assertNotEqual(d1[id_url.field(frank_1)],
d2[id_url.field(frank_2)])
self.assertEqual(d1[id_url.field(frank_1)], d1[id_url.field(frank_2)])
def doSupplierConnect(self, *args, **kwargs):
"""
Action method.
"""
from customer import supplier_connector
from customer import fire_hire
from raid import eccmap
position = self.family_position
if position is None or position == -1:
lg.warn('position for new supplier is unknown, will "guess"')
current_suppliers = list(contactsdb.suppliers())
for i in range(len(current_suppliers)):
supplier_idurl = current_suppliers[i].to_bin()
if not supplier_idurl:
position = i
break
if id_url.is_in(supplier_idurl,
fire_hire.A().dismiss_list,
as_field=False):
position = i
break
sc = supplier_connector.by_idurl(self.target_idurl)
if not sc:
sc = supplier_connector.create(
supplier_idurl=self.target_idurl,
customer_idurl=my_id.getIDURL(),
)
sc.set_callback('supplier_finder', self._supplier_connector_state)
sc.automat(
'connect',
family_position=position,
ecc_map=(self.ecc_map or eccmap.Current().name),
family_snapshot=self.family_snapshot,
)
def _on_identity_cached(self, idurl, node):
if self.stopped:
return None
if not idurl:
self._on_node_process_failed(None, node)
return None
if id_url.is_in(idurl, self.ignore_idurls):
if _Debug:
lg.dbg(
_DebugLevel,
'lookup.DiscoveryTask[%r]._on_identity_cached IGNORE %r' %
(self.id, idurl))
self._on_node_process_failed(None, node)
return None
self.cached_count += 1
idurl = id_url.to_bin(idurl)
if idurl not in discovered_idurls(layer_id=self.layer_id):
discovered_idurls(layer_id=self.layer_id).append(idurl)
known_idurls()[idurl] = time.time()
self._on_node_succeed(node, idurl)
if _Debug:
lg.out(
_DebugLevel,
'lookup.DiscoveryTask[%r]._on_identity_cached : %s' %
(self.id, idurl))
return idurl
def SendToIDs(idlist, wide=False, ack_handler=None, timeout_handler=None, response_timeout=20):
"""
Same, but send to many IDs and also check previous packets to not re-send.
"""
global _PropagateCounter
if _Debug:
lg.out(_DebugLevel, "propagate.SendToIDs to %d users, wide=%s" % (len(idlist), wide))
if ack_handler is None:
ack_handler = HandleAck
if timeout_handler is None:
timeout_handler = HandleTimeOut
LocalIdentity = my_id.getLocalIdentity()
Payload = strng.to_bin(LocalIdentity.serialize())
alreadysent = set()
totalsent = 0
inqueue = {}
found_previous_packets = 0
for pkt_out in packet_out.queue():
if id_url.is_in(pkt_out.remote_idurl, idlist, as_field=False):
if pkt_out.description.count('Identity'):
if pkt_out.remote_idurl not in inqueue:
inqueue[pkt_out.remote_idurl] = 0
inqueue[pkt_out.remote_idurl] += 1
found_previous_packets += 1
for contact in idlist:
if not contact:
continue
if contact in alreadysent:
# just want to send once even if both customer and supplier
continue
if contact in inqueue and inqueue[contact] > 2:
# now only 2 protocols is working: tcp and udp
if _Debug:
lg.out(_DebugLevel, ' skip sending [Identity] to %s, packet already in the queue' % contact)
continue
p = signed.Packet(
Command=commands.Identity(),
OwnerID=my_id.getLocalID(),
CreatorID=my_id.getLocalID(),
PacketID=('propagate:%d:%s' % (_PropagateCounter, packetid.UniqueID())),
Payload=Payload,
RemoteID=contact,
)
_PropagateCounter += 1
if _Debug:
lg.out(_DebugLevel, " sending [Identity] to %s" % nameurl.GetName(contact))
gateway.outbox(p, wide, response_timeout=response_timeout, callbacks={
commands.Ack(): ack_handler,
commands.Fail(): ack_handler,
None: timeout_handler,
})
if wide:
# this is a ping packet - need to clear old info
p2p_stats.ErasePeerProtosStates(contact)
p2p_stats.EraseMyProtosStates(contact)
alreadysent.add(contact)
totalsent += 1
del alreadysent
return totalsent
def test_three_sources(self):
self._cache_identity('george')
d3 = {
id_url.field(george_1),
id_url.field(george_2),
}
self.assertEqual(len(d3), 1)
self.assertEqual(id_url.field(george_2), id_url.field(george_1))
self.assertEqual(id_url.field(george_3), id_url.field(george_1))
self.assertIn(id_url.field(george_1), d3)
self.assertIn(id_url.field(george_2), d3)
self.assertIn(id_url.field(george_3), d3)
self._cache_identity('frank')
self.assertNotIn(id_url.field(frank_1), d3)
self.assertNotIn(id_url.field(frank_2), d3)
self.assertFalse(id_url.is_in(frank_1, d3))
self.assertTrue(id_url.is_in(george_1, d3))
self.assertTrue(id_url.is_in(george_2, d3))
self.assertTrue(id_url.is_in(george_3, d3))
def doFindNewSupplier(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r'
%
(settings.getSuppliersNumberDesired(), contactsdb.suppliers()))
from p2p import network_connector
if network_connector.A().state != 'CONNECTED':
if _Debug:
lg.out(
_DebugLevel,
' network_connector is not CONNECTED at the moment, SKIP'
)
self.automat('search-failed')
return
position_for_new_supplier = None
for pos in range(settings.getSuppliersNumberDesired()):
if pos in self.hire_list:
continue
supplier_idurl = contactsdb.supplier(pos)
if not supplier_idurl:
lg.info(
'found empty supplier at position %d and going to find new supplier on that position'
% pos)
position_for_new_supplier = pos
break
if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False):
lg.info(
'going to find new supplier on existing position %d to replace supplier %s'
% (
pos,
supplier_idurl,
))
position_for_new_supplier = pos
break
if position_for_new_supplier is None:
lg.err('did not found position for new supplier')
self.automat('search-failed')
return
from customer import supplier_finder
for idurl_txt in strng.to_text(config.conf().getData(
'services/employer/candidates')).split(','):
if idurl_txt.strip():
supplier_finder.AddSupplierToHire(idurl_txt)
self.hire_list.append(position_for_new_supplier)
supplier_finder.A(
'start',
family_position=position_for_new_supplier,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(contactsdb.suppliers()),
)
def doCloseConnectors(self, *args, **kwargs):
"""
Action method.
"""
from customer import supplier_connector
for supplier_idurl in self.dismiss_list:
sc = supplier_connector.by_idurl(supplier_idurl)
if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False):
self.dismiss_list.remove(supplier_idurl)
if sc:
sc.automat('shutdown')
def doCloseConnector(self, *args, **kwargs):
"""
Action method.
"""
from customer import supplier_connector
supplier_idurl, _ = args[0]
supplier_idurl = id_url.field(supplier_idurl)
if _Debug:
lg.args(_DebugLevel, supplier_idurl=supplier_idurl, dismiss_list=self.dismiss_list)
sc = supplier_connector.by_idurl(supplier_idurl)
if id_url.is_in(supplier_idurl.to_bin(), self.dismiss_list, as_field=False):
self.dismiss_list.remove(supplier_idurl)
if sc:
sc.automat('shutdown')
else:
raise Exception('supplier_connector must exist')
def test_in_dict(self):
self._cache_identity('alice')
self._cache_identity('bob')
self._cache_identity('carl')
d = {}
d[id_url.field(alice_text)] = 'alice'
d[id_url.field(bob)] = 'bob'
self.assertIn(id_url.field(alice_text), d)
self.assertNotIn(id_url.field(carl), d)
self.assertTrue(id_url.is_in(alice_text, d))
self.assertTrue(id_url.is_not_in(carl, d))
d2 = {
id_url.field(bob): 'bob',
'some_key': 'some_value',
}
self.assertIn('some_key', d2)
keys = list(d2.keys())
with self.assertRaises(TypeError):
(keys[0] != keys[1])
def _nodes_lookup_finished(self, idurls):
if _Debug:
lg.out(_DebugLevel,
'supplier_finder._nodes_lookup_finished : %r' % idurls)
if not idurls:
lg.warn('no available nodes found via DHT lookup')
self.automat('users-not-found')
return
found_idurl = None
for idurl in idurls:
if id_url.is_in(idurl, contactsdb.suppliers(), as_field=False):
if _Debug:
lg.out(' skip %r because already my supplier' % idurl)
continue
ident = identitycache.FromCache(idurl)
remoteprotos = set(ident.getProtoOrder())
myprotos = set(my_id.getLocalIdentity().getProtoOrder())
if not len(myprotos.intersection(remoteprotos)):
if _Debug:
lg.out(
_DebugLevel,
' skip %r because no matching protocols exists' %
idurl)
continue
found_idurl = idurl
break
if not found_idurl:
lg.warn(
'found some nodes via DHT lookup, but none of them is available'
)
self.automat('users-not-found')
return
if _Debug:
lg.out(
_DebugLevel,
' selected %r and will request supplier service' %
found_idurl)
self.automat('found-one-user', found_idurl)
def on_identity_url_changed(evt):
from access import group_member
service_dir = settings.ServiceDir('service_private_groups')
groups_dir = os.path.join(service_dir, 'groups')
brokers_dir = os.path.join(service_dir, 'brokers')
old_idurl = id_url.field(evt.data['old_idurl'])
new_idurl = id_url.field(evt.data['new_idurl'])
active_group_keys = list(active_groups())
to_be_reconnected = []
for group_key_id in active_group_keys:
if not group_key_id:
continue
group_creator_idurl = global_id.glob2idurl(group_key_id)
if id_url.is_the_same(group_creator_idurl, old_idurl):
old_group_path = os.path.join(groups_dir, group_key_id)
latest_group_key_id = my_keys.latest_key_id(group_key_id)
latest_group_path = os.path.join(groups_dir, latest_group_key_id)
lg.info('going to rename rotated group file: %r -> %r' % (old_group_path, latest_group_path, ))
if os.path.isfile(old_group_path):
try:
os.rename(old_group_path, latest_group_path)
except:
lg.exc()
continue
else:
lg.warn('key file %r was not found, key was not renamed' % old_group_path)
active_groups()[latest_group_key_id] = active_groups().pop(group_key_id)
group_member.rotate_active_group_memeber(group_key_id, latest_group_key_id)
gm = group_member.get_active_group_member(group_key_id)
if gm and gm.connected_brokers and id_url.is_in(old_idurl, gm.connected_brokers.values()):
lg.info('connected broker %r IDURL is rotated, going to reconnect %r' % (old_idurl, gm, ))
if group_key_id not in to_be_reconnected:
to_be_reconnected.append(group_key_id)
known_customers = list(known_brokers().keys())
for customer_id in known_customers:
latest_customer_id = global_id.idurl2glob(new_idurl)
customer_idurl = global_id.glob2idurl(customer_id)
if id_url.is_the_same(customer_idurl, old_idurl):
latest_customer_dir = os.path.join(brokers_dir, latest_customer_id)
lg.info('going to rename rotated customer id: %r -> %r' % (customer_id, latest_customer_id, ))
old_customer_dir = os.path.join(brokers_dir, customer_id)
if os.path.isdir(old_customer_dir):
try:
bpio.move_dir_recursive(old_customer_dir, latest_customer_dir)
bpio.rmdir_recursive(old_customer_dir)
except:
lg.exc()
continue
known_brokers()[latest_customer_id] = known_brokers().pop(customer_id)
for broker_pos, broker_id in enumerate(known_brokers(latest_customer_id)):
if not broker_id:
continue
broker_idurl = global_id.glob2idurl(broker_id)
if broker_idurl == old_idurl:
latest_broker_id = global_id.idurl2glob(new_idurl)
latest_broker_path = os.path.join(latest_customer_dir, latest_broker_id)
lg.info('going to rename rotated broker id: %r -> %r' % (broker_id, latest_broker_id, ))
old_broker_path = os.path.join(latest_customer_dir, broker_id)
if os.path.isfile(old_broker_path):
try:
os.rename(old_broker_path, latest_broker_path)
except:
lg.exc()
continue
if latest_broker_id in known_brokers(latest_customer_id):
lg.warn('broker %r already exist' % latest_broker_id)
continue
known_brokers()[latest_customer_id][broker_pos] = latest_broker_id
if _Debug:
lg.args(_DebugLevel, to_be_reconnected=to_be_reconnected)
for group_key_id in to_be_reconnected:
gm = group_member.get_active_group_member(group_key_id)
if gm:
gm.automat('reconnect')
def doSubstituteSupplier(self, *args, **kwargs):
"""
Action method.
"""
new_idurl = id_url.field(args[0])
family_position = kwargs.get('family_position')
current_suppliers = list(contactsdb.suppliers())
desired_suppliers = settings.getSuppliersNumberDesired()
old_idurl = None
if family_position in self.hire_list:
self.hire_list.remove(family_position)
lg.info(
'found position on which new supplier suppose to be hired: %d'
% family_position)
else:
lg.warn('did not found position for new supplier to be hired on')
if new_idurl in current_suppliers:
raise Exception('%s is already supplier' % new_idurl)
if family_position is None or family_position == -1:
lg.warn(
'unknown family_position from supplier results, will pick first empty spot'
)
position = -1
old_idurl = None
for i in range(len(current_suppliers)):
if not current_suppliers[i].strip():
position = i
break
if id_url.is_in(current_suppliers[i],
self.dismiss_list,
as_field=False):
position = i
old_idurl = current_suppliers[i]
break
family_position = position
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doSubstituteSupplier family_position=%d' %
family_position)
contactsdb.add_supplier(idurl=new_idurl, position=family_position)
contactsdb.save_suppliers()
misc.writeSupplierData(
new_idurl,
'connected',
time.strftime('%d-%m-%Y %H:%M:%S'),
my_id.getIDURL(),
)
from main import control
control.on_suppliers_changed(current_suppliers)
if family_position < 0:
lg.info(
'added new supplier, family position unknown: %s desired_suppliers=%d current_suppliers=%d'
% (new_idurl, desired_suppliers, len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=None,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
else:
if old_idurl:
lg.info(
'hired new supplier and substitute existing supplier on position %d : %s->%s desired_suppliers=%d current_suppliers=%d'
% (family_position, old_idurl, new_idurl,
desired_suppliers, len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=old_idurl,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
else:
lg.info(
'hired new supplier on empty position %d : %s desired_suppliers=%d current_suppliers=%d'
% (family_position, new_idurl, desired_suppliers,
len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=None,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
self.restart_interval = 1.0
if _Debug:
lg.out(_DebugLevel,
' my current suppliers: %r' % contactsdb.suppliers())
def test_empty(self):
self.assertTrue(id_url.is_empty(id_url.field(b'')))
self.assertTrue(id_url.is_empty(id_url.field('')))
self.assertTrue(id_url.is_empty(id_url.field(None)))
self.assertTrue(id_url.is_empty(id_url.field(b'None')))
self.assertTrue(id_url.is_empty(None))
self.assertTrue(id_url.is_empty(b''))
self.assertTrue(id_url.is_empty(''))
self.assertTrue(id_url.is_empty(b'None'))
self.assertTrue(id_url.is_empty('None'))
self.assertFalse(id_url.field(b''))
self.assertFalse(id_url.field(''))
self.assertFalse(id_url.field(None))
self.assertFalse(id_url.field('None'))
self.assertFalse(id_url.field(b'None'))
self.assertTrue(bool(id_url.field(b'')) is False)
self.assertTrue(bool(id_url.field('')) is False)
self.assertTrue(bool(id_url.field(None)) is False)
self.assertTrue(bool(id_url.field('None')) is False)
self.assertTrue(bool(id_url.field(b'None')) is False)
self.assertTrue(id_url.field(b'') is not None)
self.assertTrue(id_url.field('') is not None)
self.assertFalse(id_url.field(None) is None)
self.assertTrue(id_url.field(None) is not None)
self.assertTrue(id_url.field(b'') == b'')
self.assertTrue(id_url.field('') == '')
self.assertTrue(id_url.field('') == b'')
self.assertTrue(id_url.field(None) == '')
self.assertTrue(id_url.field(None) == b'')
l = [
b'',
None,
'',
id_url.field(''),
id_url.field(None),
id_url.field(b''),
]
self.assertIn(b'', l)
self.assertIn('', l)
self.assertIn(None, l)
self.assertTrue(id_url.is_some_empty(l))
self.assertEqual(l.count(None), 4)
self.assertEqual(id_url.empty_count(l), 6)
self.assertTrue(None in [
id_url.field(None),
])
self.assertTrue(None in [
id_url.field(b''),
])
self.assertTrue(b'' in [
id_url.field(None),
])
self.assertTrue(b'' in [
id_url.field(b''),
])
self.assertTrue(id_url.is_in(None, [
id_url.field(None),
]))
self.assertTrue(id_url.is_in(None, [
id_url.field(b''),
]))
self.assertTrue(id_url.is_in(b'', [
id_url.field(None),
]))
self.assertTrue(id_url.is_in(b'', [
id_url.field(b''),
]))
d = {id_url.field(''): 0, id_url.field(None): 1, id_url.field(b''): 2}
self.assertTrue(len(d), 1)
self.assertTrue(b'' in d)
self.assertTrue('' in d)
self.assertFalse(b'' not in d)
self.assertFalse('' not in d)
self.assertNotIn(None, d)
self.assertIn(id_url.field(''), d)
self.assertIn(id_url.field(b''), d)
self.assertIn(id_url.field(None), d)