def isStillNeeded(self, *args, **kwargs):
"""
Condition method.
"""
supplier_idurl = args[0]
current_suppliers = contactsdb.suppliers()
if supplier_idurl in current_suppliers:
# this guy is already a supplier, we still need more then
return True
desired_number = settings.getSuppliersNumberDesired()
needed_suppliers = current_suppliers[:desired_number]
empty_suppliers = needed_suppliers.count(id_url.field(b''))
# if '' in needed_suppliers:
# lg.warn('found empty suppliers!!!')
# return True
s = set(id_url.to_bin_list(needed_suppliers))
s.add(id_url.to_bin(supplier_idurl))
s.difference_update(set(id_url.to_bin_list(self.dismiss_list)))
result = len(s) - empty_suppliers < settings.getSuppliersNumberDesired(
)
# if _Debug:
# lg.out(_DebugLevel, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % (
# contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list),
# len(s), settings.getSuppliersNumberDesired(), result))
return result
def isMoreNeeded(self, *args, **kwargs):
"""
Condition method.
"""
# if _Debug:
# lg.out(_DebugLevel, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
# contactsdb.num_suppliers(), len(self.dismiss_list),
# settings.getSuppliersNumberDesired()))
if id_url.is_some_empty(contactsdb.suppliers()):
if _Debug:
lg.out(_DebugLevel,
'fire_hire.isMoreNeeded found empty supplier!!!')
return True
if isinstance(args[0], list):
dismissed = args[0]
else:
dismissed = self.dismiss_list
s = set(id_url.to_bin_list(contactsdb.suppliers()))
s.difference_update(set(id_url.to_bin_list(dismissed)))
result = len(s) < settings.getSuppliersNumberDesired()
if _Debug:
lg.out(
_DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' %
(contactsdb.num_suppliers(), len(dismissed), len(s),
settings.getSuppliersNumberDesired(), result))
return result
def is_correspondent(idurl):
"""
Return True if given ID is found in correspondents list.
"""
if id_url.is_empty(idurl):
return False
return id_url.field(idurl).to_bin() in id_url.to_bin_list(correspondents_ids())
def is_customer(idurl):
"""
Return True if given ID is found in customers list.
"""
if id_url.is_empty(idurl):
return False
return id_url.field(idurl).to_bin() in id_url.to_bin_list(customers())
def is_supplier(idurl, customer_idurl=None):
"""
Return True if given ID is found in suppliers list.
"""
if id_url.is_empty(idurl):
return False
return id_url.field(idurl).to_bin() in id_url.to_bin_list(suppliers(customer_idurl=customer_idurl))
def _on_existing_customer_accepted(self, evt):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from supplier import family_member
from userid import id_url
from userid import my_id
customer_idurl = evt.data['idurl']
if customer_idurl == my_id.getLocalID():
lg.warn('skipping my own identity')
return
if evt.data.get('position') is None:
lg.warn('position of supplier in the family is still unclear')
return
fm = family_member.by_customer_idurl(customer_idurl)
if not fm:
lg.err(
'family_member() instance was not found for existing customer %s'
% customer_idurl)
return
reactor.callLater(0, fm.automat, 'family-join', { # @UndefinedVariable
'supplier_idurl': my_id.getLocalID().to_bin(),
'ecc_map': evt.data.get('ecc_map'),
'position': evt.data.get('position'),
'family_snapshot': id_url.to_bin_list(evt.data.get('family_snapshot')),
})
def doConnectSuppliers(self, *args, **kwargs):
"""
Action method.
"""
from customer import supplier_connector
from p2p import online_status
self.connect_list = []
my_current_family = contactsdb.suppliers()
for pos, supplier_idurl in enumerate(my_current_family):
if not supplier_idurl:
continue
if self.configs[0] and pos >= self.configs[0]:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None:
sc = supplier_connector.create(
supplier_idurl=supplier_idurl,
customer_idurl=my_id.getIDURL(),
)
else:
sc.needed_bytes = None
sc.do_calculate_needed_bytes()
sc.set_callback('fire_hire',
self._on_supplier_connector_state_changed)
self.connect_list.append(supplier_idurl)
sc.automat(
'connect',
family_position=pos,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(my_current_family),
)
online_status.add_online_status_listener_callback(
idurl=supplier_idurl,
callback_method=self._on_supplier_online_status_state_changed,
)
def total_suppliers():
"""
"""
global _SuppliersList
result = set()
for suppliers_list in _SuppliersList.values():
result.update(set(id_url.to_bin_list(suppliers_list)))
return len(result)
def contacts_remote(include_all=False, include_enabled=True):
"""
Return ID's list of all known peers.
"""
allcontactslist = id_url.to_bin_list(contacts(include_all=include_all, include_enabled=include_enabled))
if my_id.getLocalID().to_bin() in allcontactslist:
allcontactslist.remove(my_id.getLocalID().to_bin())
return id_url.fields_list(allcontactslist)
def contacts_remote():
"""
Return ID's list of all known peers.
"""
allcontactslist = id_url.to_bin_list(contacts_full())
if my_id.getLocalID().to_bin() in allcontactslist:
allcontactslist.remove(my_id.getLocalID().to_bin())
return id_url.fields_list(allcontactslist)
def getSources(self, as_fields=True, as_originals=False):
"""
Return identity sources.
"""
if as_originals:
return id_url.to_original_list(self.sources)
if not as_fields:
return id_url.to_bin_list(self.sources)
return self.sources
def add_customer_meta_info(customer_idurl, info):
"""
"""
global _CustomersMetaInfo
customer_idurl = id_url.field(customer_idurl)
if not customer_idurl.is_latest():
if customer_idurl.original() in _CustomersMetaInfo:
if customer_idurl.to_bin() not in _CustomersMetaInfo:
_CustomersMetaInfo[
customer_idurl.to_bin()] = _CustomersMetaInfo.pop(
customer_idurl.original())
lg.info(
'detected and processed idurl rotate for customer meta info : %r -> %r'
% (customer_idurl.original(), customer_idurl.to_bin()))
customer_idurl = id_url.to_bin(customer_idurl)
if 'family_snapshot' in info:
info['family_snapshot'] = id_url.to_bin_list(info['family_snapshot'])
if 'ecc_map' in info:
info['ecc_map'] = strng.to_text(info['ecc_map'])
if customer_idurl not in _CustomersMetaInfo:
if _Debug:
lg.out(
_DebugLevel,
'contactsdb.add_customer_meta_info store new meta info for customer %r: %r'
% (
customer_idurl,
info,
))
_CustomersMetaInfo[customer_idurl] = {}
else:
if _Debug:
lg.out(
_DebugLevel,
'contactsdb.add_customer_meta_info update existing meta info for customer %r: %r'
% (
customer_idurl,
info,
))
_CustomersMetaInfo[customer_idurl].update(info)
json_info = {
k: jsn.dict_keys_to_text(v)
for k, v in id_url.to_bin_dict(_CustomersMetaInfo).items()
}
try:
raw_data = jsn.dumps(
json_info,
indent=2,
sort_keys=True,
keys_to_text=True,
values_to_text=True,
)
except:
lg.exc()
return None
local_fs.WriteTextFile(settings.CustomersMetaInfoFilename(), raw_data)
return _CustomersMetaInfo
def doFindNewSupplier(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r'
%
(settings.getSuppliersNumberDesired(), contactsdb.suppliers()))
from p2p import network_connector
if network_connector.A().state != 'CONNECTED':
if _Debug:
lg.out(
_DebugLevel,
' network_connector is not CONNECTED at the moment, SKIP'
)
self.automat('search-failed')
return
position_for_new_supplier = None
for pos in range(settings.getSuppliersNumberDesired()):
if pos in self.hire_list:
continue
supplier_idurl = contactsdb.supplier(pos)
if not supplier_idurl:
lg.info(
'found empty supplier at position %d and going to find new supplier on that position'
% pos)
position_for_new_supplier = pos
break
if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False):
lg.info(
'going to find new supplier on existing position %d to replace supplier %s'
% (
pos,
supplier_idurl,
))
position_for_new_supplier = pos
break
if position_for_new_supplier is None:
lg.err('did not found position for new supplier')
self.automat('search-failed')
return
from customer import supplier_finder
for idurl_txt in strng.to_text(config.conf().getData(
'services/employer/candidates')).split(','):
if idurl_txt.strip():
supplier_finder.AddSupplierToHire(idurl_txt)
self.hire_list.append(position_for_new_supplier)
supplier_finder.A(
'start',
family_position=position_for_new_supplier,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(contactsdb.suppliers()),
)
def doInit(self, *args, **kwargs):
"""
Action method.
"""
self.lookup_method = kwargs.get('lookup_method', None)
self.target_service = kwargs['target_service']
self.request_service_params = kwargs.get('request_service_params',
None)
self.result_callback = kwargs.get('result_callback', None)
self.exclude_nodes = id_url.to_bin_list(kwargs.get(
'exclude_nodes', []))
def extract_discovered_idurls(count=1, layer_id=0):
if not discovered_idurls(layer_id=layer_id):
if _Debug:
lg.out(_DebugLevel,
'lookup.extract_discovered_idurls returns empty list')
return []
discovered = list(discovered_idurls(layer_id=layer_id))
# random.shuffle(discovered)
results = id_url.to_bin_list(discovered[:count])
if _Debug:
lg.out(_DebugLevel, 'lookup.extract_discovered_idurls : %s' % results)
return results
def _do_verify(dht_value, customer_idurl_bin):
if customer_idurl_bin in rotated_idurls:
rotated_idurls.remove(customer_idurl_bin)
ret = {
'suppliers': [],
'ecc_map': None,
'customer_idurl': customer_idurl,
'revision': 0,
'publisher_idurl': None,
'timestamp': None,
}
if not dht_value or not isinstance(dht_value, dict):
if not rotated_idurls:
result.callback(ret)
return ret
another_customer_idurl_bin = rotated_idurls.pop(0)
lg.warn(
'found another rotated idurl %r and re-try reading customer suppliers'
% another_customer_idurl_bin)
d = dht_records.get_suppliers(another_customer_idurl_bin,
return_details=True,
use_cache=False)
d.addCallback(_do_verify, another_customer_idurl_bin)
d.addErrback(_on_error)
return ret
try:
_ecc_map = strng.to_text(dht_value['ecc_map'])
if as_fields:
_customer_idurl = id_url.field(dht_value['customer_idurl'])
_publisher_idurl = id_url.field(
dht_value.get('publisher_idurl'))
_suppliers_list = id_url.fields_list(dht_value['suppliers'])
else:
_customer_idurl = id_url.to_bin(dht_value['customer_idurl'])
_publisher_idurl = id_url.to_bin(
dht_value.get('publisher_idurl'))
_suppliers_list = id_url.to_bin_list(dht_value['suppliers'])
_revision = int(dht_value.get('revision'))
_timestamp = int(dht_value.get('timestamp'))
except:
lg.exc()
result.callback(ret)
return ret
ret.update({
'suppliers': _suppliers_list,
'ecc_map': _ecc_map,
'customer_idurl': _customer_idurl,
'revision': _revision,
'publisher_idurl': _publisher_idurl,
'timestamp': _timestamp,
})
return _do_identity_cache(ret)
def customer_position(idurl):
"""
Return position of supplier with given ID or -1.
"""
if not idurl:
return -1
if not id_url.is_cached(idurl):
return -1
idurl = id_url.field(idurl)
try:
index = id_url.to_bin_list(customers()).index(idurl.to_bin())
except:
index = -1
return index
def _do_request_supplier_service(self, ecc_map, family_position,
family_snapshot):
if _Debug:
lg.args(_DebugLevel,
supplier_idurl=self.supplier_idurl,
ecc_map=ecc_map,
family_position=family_position,
family_snapshot=family_snapshot)
if not self.supplier_idurl:
lg.warn(
'supplier idurl is empty, SKIP sending supplier_service request'
)
return
service_info = {
'needed_bytes': self.needed_bytes,
'customer_id': global_id.UrlToGlobalID(self.customer_idurl),
}
my_customer_key_id = my_id.getGlobalID(key_alias='customer')
if my_keys.is_key_registered(my_customer_key_id):
service_info['customer_public_key'] = my_keys.get_key_info(
key_id=my_customer_key_id,
include_private=False,
include_signature=True,
generate_signature=True,
)
else:
lg.warn('my own customer key is not registered: %r' %
my_customer_key_id)
if self.key_id:
service_info['key_id'] = self.key_id
self._last_known_ecc_map = ecc_map
if self._last_known_ecc_map is not None:
service_info['ecc_map'] = self._last_known_ecc_map
self._last_known_family_position = family_position
if self._last_known_family_position is not None:
service_info['position'] = self._last_known_family_position
self._last_known_family_snapshot = family_snapshot
if self._last_known_family_snapshot is not None:
service_info['family_snapshot'] = id_url.to_bin_list(
self._last_known_family_snapshot)
request = p2p_service.SendRequestService(
remote_idurl=self.supplier_idurl,
service_name='service_supplier',
json_payload=service_info,
callbacks={
commands.Ack(): self._supplier_service_acked,
commands.Fail(): self._supplier_service_failed,
},
)
self.request_packet_id = request.PacketID
def doInit(self, *args, **kwargs):
"""
Action method.
"""
self.lookup_method = kwargs.get('lookup_method', None)
self.target_service = kwargs['target_service']
self.request_service_params = kwargs.get('request_service_params',
None)
self.request_service_timeout = kwargs.get('request_service_timeout',
120)
self.ping_retries = kwargs.get('ping_retries', None)
self.ack_timeout = kwargs.get('ack_timeout', None)
self.force_handshake = kwargs.get('force_handshake', False)
self.result_callback = kwargs.get('result_callback', None)
self.exclude_nodes = id_url.to_bin_list(kwargs.get(
'exclude_nodes', []))
self.retries = kwargs.get('attempts', 5)
def contact_position(idurl):
"""
Return position for given contact ID in the total list combined from
suppliers, customers.
Suppliers should be numbered 0 to 63 with customers after that not
sure we can count on numbers staying.
"""
if not idurl:
return -1
if not id_url.is_cached(idurl):
return -1
idurl = id_url.field(idurl)
try:
index = id_url.to_bin_list(contacts_list()).index(idurl.to_bin())
except:
index = -1
return index
def _on_my_dht_relations_discovered(self, dht_result):
from p2p import p2p_service
from contacts import contactsdb
from userid import my_id
from userid import id_url
from crypt import my_keys
from logs import lg
if not (dht_result and isinstance(dht_result, dict)
and len(dht_result.get('suppliers', [])) > 0):
lg.warn('no dht records found for my customer family')
return
if id_url.is_some_empty(contactsdb.suppliers()):
lg.warn(
'some of my suppliers are not hired yet, skip doing any changes'
)
return
suppliers_to_be_dismissed = set()
dht_suppliers = id_url.to_bin_list(dht_result['suppliers'])
# clean up old suppliers
for idurl in dht_suppliers:
if not idurl:
continue
if not contactsdb.is_supplier(idurl):
lg.warn('dht relation with %r is not valid anymore' % idurl)
suppliers_to_be_dismissed.add(idurl)
for supplier_idurl in suppliers_to_be_dismissed:
service_info = {}
my_customer_key_id = my_id.getGlobalID(key_alias='customer')
if my_keys.is_key_registered(my_customer_key_id):
service_info['customer_public_key'] = my_keys.get_key_info(
key_id=my_customer_key_id,
include_private=False,
include_signature=False,
generate_signature=False,
)
p2p_service.SendCancelService(
remote_idurl=supplier_idurl,
service_name='service_supplier',
json_payload=service_info,
)
if suppliers_to_be_dismissed:
lg.info(
'found %d suppliers to be cleaned and sent CancelService() packets'
% len(suppliers_to_be_dismissed))
def _on_new_customer_accepted(self, evt):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from userid import my_id
from userid import id_url
from supplier import family_member
customer_idurl = evt.data['idurl']
fm = family_member.by_customer_idurl(customer_idurl)
if not fm:
fm = family_member.create_family(customer_idurl)
fm.automat('init')
else:
lg.warn(
'family_member() instance already exists, but new customer just accepted %s'
% customer_idurl)
reactor.callLater(0, fm.automat, 'family-join', { # @UndefinedVariable
'supplier_idurl': my_id.getLocalID().to_bin(),
'ecc_map': evt.data.get('ecc_map'),
'position': evt.data.get('position', -1),
'family_snapshot': id_url.to_bin_list(evt.data.get('family_snapshot')),
})
def start(self):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from contacts import contactsdb
from userid import id_url
from supplier import family_member
from transport import callback
from userid import my_id
callback.append_inbox_callback(self._on_inbox_packet_received)
for customer_idurl in contactsdb.customers():
if not customer_idurl:
continue
if not id_url.is_cached(customer_idurl):
continue
if customer_idurl == my_id.getLocalID():
lg.warn('skipping my own identity')
continue
fm = family_member.by_customer_idurl(customer_idurl)
if not fm:
fm = family_member.create_family(customer_idurl)
fm.automat('init')
local_customer_meta_info = contactsdb.get_customer_meta_info(
customer_idurl)
reactor.callLater(0, fm.automat, 'family-join', { # @UndefinedVariable
'supplier_idurl': my_id.getLocalID().to_bin(),
'ecc_map': local_customer_meta_info.get('ecc_map'),
'position': local_customer_meta_info.get('position', -1),
'family_snapshot': id_url.to_bin_list(local_customer_meta_info.get('family_snapshot')),
})
events.add_subscriber(self._on_identity_url_changed,
'identity-url-changed')
events.add_subscriber(self._on_existing_customer_accepted,
'existing-customer-accepted')
events.add_subscriber(self._on_new_customer_accepted,
'new-customer-accepted')
events.add_subscriber(self._on_existing_customer_terminated,
'existing-customer-terminated')
return True
def _do_verify(dht_value):
ret = {
'suppliers': [],
'ecc_map': None,
'customer_idurl': customer_idurl,
'revision': 0,
'publisher_idurl': None,
'timestamp': None,
}
if not dht_value or not isinstance(dht_value, dict):
result.callback(ret)
return ret
try:
_ecc_map = strng.to_text(dht_value['ecc_map'])
if as_fields:
_customer_idurl = id_url.field(dht_value['customer_idurl'])
_publisher_idurl = id_url.field(
dht_value.get('publisher_idurl'))
_suppliers_list = id_url.fields_list(dht_value['suppliers'])
else:
_customer_idurl = id_url.to_bin(dht_value['customer_idurl'])
_publisher_idurl = id_url.to_bin(
dht_value.get('publisher_idurl'))
_suppliers_list = id_url.to_bin_list(dht_value['suppliers'])
_revision = int(dht_value.get('revision'))
_timestamp = int(dht_value.get('timestamp'))
except:
lg.exc()
result.callback(ret)
return ret
ret.update({
'suppliers': _suppliers_list,
'ecc_map': _ecc_map,
'customer_idurl': _customer_idurl,
'revision': _revision,
'publisher_idurl': _publisher_idurl,
'timestamp': _timestamp,
})
return _do_identity_cache(ret)
def _on_my_dht_relations_discovered(self, dht_result):
if not (dht_result and isinstance(dht_result, dict)
and len(dht_result.get('suppliers', [])) > 0):
lg.warn('no dht records found for my customer family')
self.automat('suppliers-read-failed')
return
dht_suppliers = id_url.to_bin_list(dht_result['suppliers'])
dht_ecc_map = dht_result.get('ecc_map', settings.DefaultEccMapName())
try:
dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber(
dht_ecc_map)
except:
lg.exc()
dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber(
settings.DefaultEccMapName())
settings.config.conf().setInt('services/customer/suppliers-number',
dht_desired_suppliers_number)
contactsdb.set_suppliers(dht_suppliers)
contactsdb.save_suppliers()
lg.info('found and restored list of %d suppliers from DHT' %
dht_desired_suppliers_number)
self.automat('suppliers-read-ok')
def validate_customers_quotas(space_dict=None, free_space=None):
unknown_customers = set()
unused_quotas = set()
if space_dict is None or free_space is None:
space_dict, free_space = read_customers_quotas()
for idurl in list(space_dict.keys()):
idurl = strng.to_bin(idurl)
try:
space_dict[idurl] = int(space_dict[idurl])
except:
unknown_customers.add(idurl)
continue
if space_dict[idurl] <= 0:
unknown_customers.add(idurl)
continue
for idurl in contactsdb.customers():
if idurl.to_bin() not in list(space_dict.keys()):
unknown_customers.add(idurl)
for idurl in space_dict.keys():
if idurl not in id_url.to_bin_list(contactsdb.customers()):
unused_quotas.add(idurl)
return unknown_customers, unused_quotas
def doSubstituteSupplier(self, *args, **kwargs):
"""
Action method.
"""
new_idurl = id_url.field(args[0])
family_position = kwargs.get('family_position')
current_suppliers = list(contactsdb.suppliers())
desired_suppliers = settings.getSuppliersNumberDesired()
old_idurl = None
if family_position in self.hire_list:
self.hire_list.remove(family_position)
lg.info(
'found position on which new supplier suppose to be hired: %d'
% family_position)
else:
lg.warn('did not found position for new supplier to be hired on')
if new_idurl in current_suppliers:
raise Exception('%s is already supplier' % new_idurl)
if family_position is None or family_position == -1:
lg.warn(
'unknown family_position from supplier results, will pick first empty spot'
)
position = -1
old_idurl = None
for i in range(len(current_suppliers)):
if not current_suppliers[i].strip():
position = i
break
if id_url.is_in(current_suppliers[i],
self.dismiss_list,
as_field=False):
position = i
old_idurl = current_suppliers[i]
break
family_position = position
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doSubstituteSupplier family_position=%d' %
family_position)
contactsdb.add_supplier(idurl=new_idurl, position=family_position)
contactsdb.save_suppliers()
misc.writeSupplierData(
new_idurl,
'connected',
time.strftime('%d-%m-%Y %H:%M:%S'),
my_id.getIDURL(),
)
from main import control
control.on_suppliers_changed(current_suppliers)
if family_position < 0:
lg.info(
'added new supplier, family position unknown: %s desired_suppliers=%d current_suppliers=%d'
% (new_idurl, desired_suppliers, len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=None,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
else:
if old_idurl:
lg.info(
'hired new supplier and substitute existing supplier on position %d : %s->%s desired_suppliers=%d current_suppliers=%d'
% (family_position, old_idurl, new_idurl,
desired_suppliers, len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=old_idurl,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
else:
lg.info(
'hired new supplier on empty position %d : %s desired_suppliers=%d current_suppliers=%d'
% (family_position, new_idurl, desired_suppliers,
len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=None,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
self.restart_interval = 1.0
if _Debug:
lg.out(_DebugLevel,
' my current suppliers: %r' % contactsdb.suppliers())
def request(self, json_payload, newpacket, info):
from twisted.internet import reactor # @UnresolvedImport
from logs import lg
from main import events
from crypt import my_keys
from p2p import p2p_service
from contacts import contactsdb
from storage import accounting
from supplier import customer_space
from userid import id_url
from userid import global_id
customer_idurl = newpacket.OwnerID
customer_id = global_id.UrlToGlobalID(customer_idurl)
bytes_for_customer = 0
try:
bytes_for_customer = int(json_payload['needed_bytes'])
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'invalid payload')
try:
customer_public_key = json_payload['customer_public_key']
customer_public_key_id = customer_public_key['key_id']
except:
customer_public_key = None
customer_public_key_id = None
data_owner_idurl = None
target_customer_idurl = None
family_position = json_payload.get('position')
ecc_map = json_payload.get('ecc_map')
family_snapshot = json_payload.get('family_snapshot')
if family_snapshot:
family_snapshot = id_url.to_bin_list(family_snapshot)
key_id = json_payload.get('key_id')
key_id = my_keys.latest_key_id(key_id)
target_customer_id = json_payload.get('customer_id')
if key_id:
# this is a request from external user to access shared data stored by one of my customers
# this is "second" customer requesting data from "first" customer
if not key_id or not my_keys.is_valid_key_id(key_id):
lg.warn('missed or invalid key id')
return p2p_service.SendFail(newpacket, 'invalid key id')
target_customer_idurl = global_id.GlobalUserToIDURL(
target_customer_id)
if not contactsdb.is_customer(target_customer_idurl):
lg.warn("target user %s is not a customer" %
target_customer_id)
return p2p_service.SendFail(newpacket, 'not a customer')
if target_customer_idurl == customer_idurl:
lg.warn('customer %s requesting shared access to own files' %
customer_idurl)
return p2p_service.SendFail(newpacket, 'invalid case')
if not my_keys.is_key_registered(key_id):
lg.warn('key not registered: %s' % key_id)
p2p_service.SendFail(newpacket, 'key not registered')
return False
data_owner_idurl = my_keys.split_key_id(key_id)[1]
if data_owner_idurl != target_customer_idurl and data_owner_idurl != customer_idurl:
# pretty complex scenario:
# external customer requesting access to data which belongs not to that customer
# this is "third" customer accessing data belongs to "second" customer
# TODO: for now just stop it
lg.warn(
'under construction, key_id=%s customer_idurl=%s target_customer_idurl=%s'
% (
key_id,
customer_idurl,
target_customer_idurl,
))
p2p_service.SendFail(newpacket, 'under construction')
return False
customer_space.register_customer_key(customer_public_key_id,
customer_public_key)
# do not create connection with that customer, only accept the request
lg.info(
'external customer %s requested access to shared data at %s' %
(
customer_id,
key_id,
))
return p2p_service.SendAck(newpacket, 'accepted')
# key_id is not present in the request:
# this is a request to connect new customer (or reconnect existing one) to that supplier
if not bytes_for_customer or bytes_for_customer < 0:
lg.warn("wrong payload : %s" % newpacket.Payload)
return p2p_service.SendFail(newpacket, 'wrong storage value')
current_customers = contactsdb.customers()
if accounting.check_create_customers_quotas():
lg.info('created new customers quotas file')
space_dict, free_space = accounting.read_customers_quotas()
try:
free_bytes = int(free_space)
except:
lg.exc()
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl not in current_customers
and customer_idurl.to_bin() in list(space_dict.keys())):
lg.warn("broken space file")
return p2p_service.SendFail(newpacket, 'broken space file')
if (customer_idurl in current_customers
and customer_idurl.to_bin() not in list(space_dict.keys())):
# seems like customer's idurl was rotated, but space file still have the old idurl
# need to find that old idurl value and replace with the new one
for other_customer_idurl in space_dict.keys():
if other_customer_idurl and other_customer_idurl != 'free' and id_url.field(
other_customer_idurl) == customer_idurl:
lg.info(
'found rotated customer identity in space file, switching: %r -> %r'
% (other_customer_idurl, customer_idurl.to_bin()))
space_dict[customer_idurl.to_bin()] = space_dict.pop(
other_customer_idurl)
break
if customer_idurl.to_bin() not in list(space_dict.keys()):
lg.warn("broken customers file")
return p2p_service.SendFail(newpacket, 'broken customers file')
if customer_idurl in current_customers:
free_bytes += int(space_dict.get(customer_idurl.to_bin(), 0))
current_customers.remove(customer_idurl)
space_dict.pop(customer_idurl.to_bin())
new_customer = False
else:
new_customer = True
lg.args(8,
new_customer=new_customer,
current_allocated_bytes=space_dict.get(
customer_idurl.to_bin()))
from supplier import local_tester
if free_bytes <= bytes_for_customer:
contactsdb.remove_customer_meta_info(customer_idurl)
accounting.write_customers_quotas(space_dict, free_bytes)
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
if customer_public_key_id:
my_keys.erase_key(customer_public_key_id)
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.info("NEW CUSTOMER: DENIED not enough space available")
events.send('new-customer-denied',
data=dict(idurl=customer_idurl))
else:
lg.info("OLD CUSTOMER: DENIED not enough space available")
events.send('existing-customer-denied',
data=dict(idurl=customer_idurl))
return p2p_service.SendAck(newpacket, 'deny')
free_bytes = free_bytes - bytes_for_customer
current_customers.append(customer_idurl)
space_dict[customer_idurl.to_bin()] = bytes_for_customer
contactsdb.add_customer_meta_info(
customer_idurl, {
'ecc_map': ecc_map,
'position': family_position,
'family_snapshot': family_snapshot,
})
accounting.write_customers_quotas(space_dict, free_bytes)
contactsdb.update_customers(current_customers)
contactsdb.save_customers()
customer_space.register_customer_key(customer_public_key_id,
customer_public_key)
reactor.callLater(
0, local_tester.TestUpdateCustomers) # @UndefinedVariable
if new_customer:
lg.info(
"NEW CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
events.send('new-customer-accepted',
data=dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
family_snapshot=family_snapshot,
key_id=customer_public_key_id,
))
else:
lg.info(
"OLD CUSTOMER: ACCEPTED %s family_position=%s ecc_map=%s allocated_bytes=%s"
%
(customer_idurl, family_position, ecc_map, bytes_for_customer))
events.send('existing-customer-accepted',
data=dict(
idurl=customer_idurl,
allocated_bytes=bytes_for_customer,
ecc_map=ecc_map,
position=family_position,
key_id=customer_public_key_id,
family_snapshot=family_snapshot,
))
return p2p_service.SendAck(newpacket, 'accepted')
def doHireNewBroker(self, event, *args, **kwargs):
"""
Action method.
"""
known_brokers = {}
known_brokers.update(self.cooperated_brokers or {})
known_brokers[self.desired_position] = self.my_broker_idurl
target_pos = self.desired_position - 1
exclude_brokers = list(
id_url.to_bin_list(filter(None, self.dht_brokers.values())))
exclude_brokers.extend(
list(
id_url.to_bin_list(
filter(None, self.requestor_known_brokers.values()))))
preferred_brokers = []
preferred_brokers_raw = config.conf().getData(
'services/message-broker/preferred-brokers').strip()
if preferred_brokers_raw:
preferred_brokers_list = re.split('\n|,|;| ',
preferred_brokers_raw)
preferred_brokers.extend(preferred_brokers_list)
preferred_brokers = id_url.to_bin_list(preferred_brokers)
if preferred_brokers:
preferred_brokers = [
x for x in preferred_brokers if x not in exclude_brokers
]
if _Debug:
lg.args(_DebugLevel,
e=event,
my=self.my_position,
desired=self.desired_position,
target=target_pos,
exclude=exclude_brokers,
preferred=preferred_brokers)
if preferred_brokers:
preferred_broker_idurl = id_url.field(preferred_brokers[0])
if preferred_broker_idurl and id_url.is_not_in(
preferred_broker_idurl, exclude_brokers, as_field=False):
result = p2p_service_seeker.connect_known_node(
remote_idurl=preferred_broker_idurl,
service_name='service_message_broker',
service_params=lambda idurl: self
._do_prepare_service_request_params(
idurl, target_pos, known_brokers, event),
request_service_timeout=self.broker_negotiate_ack_timeout *
(target_pos + 1),
exclude_nodes=list(exclude_brokers),
force_handshake=True,
attempts=1,
)
result.addCallback(self._on_new_broker_hired, target_pos,
self.my_position, self.desired_position)
if _Debug:
result.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='broker_negotiator.doHirePrevBroker')
result.addErrback(self._on_new_broker_lookup_failed,
target_pos)
return
result = p2p_service_seeker.connect_random_node(
lookup_method=lookup.random_message_broker,
service_name='service_message_broker',
service_params=lambda
idurl: self._do_prepare_service_request_params(
idurl, target_pos, known_brokers, event),
request_service_timeout=self.broker_negotiate_ack_timeout *
(target_pos + 1),
exclude_nodes=list(exclude_brokers),
attempts=1,
force_handshake=True,
)
result.addCallback(self._on_new_broker_hired, target_pos,
self.my_position, self.desired_position)
if _Debug:
result.addErrback(lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='broker_negotiator.doHirePrevBroker')
result.addErrback(self._on_new_broker_lookup_failed, target_pos)
def report_donated_storage():
space_dict, free_space = read_customers_quotas()
used_space_dict = read_customers_usage()
r = {}
r['customers_num'] = contactsdb.num_customers()
r['customers'] = []
r['old_customers'] = []
r['errors'] = []
r['consumed'] = 0
r['donated'] = settings.getDonatedBytes()
# r['donated_str'] = diskspace.MakeStringFromBytes(r['donated'])
r['real'] = bpio.getDirectorySize(settings.getCustomersFilesDir())
try:
r['free'] = int(free_space)
except:
r['free'] = 0
used = 0
for idurl in id_url.to_bin_list(contactsdb.customers()):
consumed_by_customer = 0
used_by_customer = 0
if idurl not in list(space_dict.keys()):
r['errors'].append('space consumed by customer %r is unknown' %
idurl)
else:
try:
consumed_by_customer = int(space_dict.pop(idurl))
r['consumed'] += consumed_by_customer
except:
r['errors'].append(
'incorrect value of consumed space for customer %r' %
idurl)
continue
if idurl in list(used_space_dict.keys()):
try:
used_by_customer = int(used_space_dict.pop(idurl))
used += used_by_customer
except:
r['errors'].append(
'incorrect value of used space for customer %r' % idurl)
continue
if consumed_by_customer < used_by_customer:
r['errors'].append(
'customer %r currently using more space than requested' %
idurl)
c = {}
c['idurl'] = strng.to_text(idurl)
c['used'] = used_by_customer
# c['used_str'] = diskspace.MakeStringFromBytes(c['used'])
c['consumed'] = consumed_by_customer
# c['consumed_str'] = diskspace.MakeStringFromBytes(c['consumed'])
c['real'] = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
# c['real_str'] = diskspace.MakeStringFromBytes(c['real'])
r['customers'].append(c)
r['used'] = used
# r['used_str'] = diskspace.MakeStringFromBytes(r['used'])
# r['consumed_str'] = diskspace.MakeStringFromBytes(r['consumed'])
if r['donated'] != r['free'] + r['consumed']:
r['errors'].append(
'total consumed %d and known free %d (%d total) bytes not match with donated %d bytes'
% (r['consumed'], r['free'], r['consumed'] + r['free'],
r['donated']))
if r['used'] > r['donated']:
r['errors'].append(
'total space used by customers exceed the donated limit')
if len(space_dict) > 0:
r['errors'].append('found %d incorrect records of consumed space' %
len(space_dict))
if r['real'] != r['used']:
r['errors'].append(
'current info needs update, known size is %d bytes but real is %d bytes'
% (r['used'], r['real']))
old_customers_used = 0
old_customers_real = 0
for idurl in used_space_dict.keys():
real = bpio.getDirectorySize(settings.getCustomerFilesDir(idurl))
try:
used = int(used_space_dict[idurl])
except:
r['errors'].append(
'incorrect value of used space for customer %r' % idurl)
continue
r['old_customers'].append({
'idurl': strng.to_text(idurl),
'used': used,
# 'used_str': diskspace.MakeStringFromBytes(used_space_dict[idurl]),
'real': real,
# 'real_str': diskspace.MakeStringFromBytes(real),
})
old_customers_used += used
old_customers_real += real
r['old_customers_used'] = old_customers_used
r['old_customers_real'] = old_customers_real
try:
r['used_percent'] = misc.value2percent(float(r['used']),
float(r['donated']), 5)
except:
r['used_percent'] = ''
try:
r['consumed_percent'] = misc.value2percent(float(r['consumed']),
float(r['donated']), 5)
except:
r['consumed_percent'] = ''
return r
