def __init__(self,
supplier_idurl,
customer_idurl,
needed_bytes,
key_id=None,
queue_subscribe=True):
"""
"""
self.supplier_idurl = supplier_idurl
self.customer_idurl = customer_idurl
self.needed_bytes = needed_bytes
self.key_id = key_id
self.queue_subscribe = queue_subscribe
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(
settings.getNeededString(), 0)
num_suppliers = -1
if self.customer_idurl == my_id.getLocalIDURL():
num_suppliers = settings.getSuppliersNumberDesired()
else:
known_ecc_map = contactsdb.get_customer_meta_info(
customer_idurl).get('ecc_map')
if known_ecc_map:
num_suppliers = eccmap.GetEccMapSuppliersNumber(
known_ecc_map)
if num_suppliers > 0:
self.needed_bytes = int(
math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
raise Exception(
'not possible to determine needed_bytes value to be requested from that supplier'
)
# self.needed_bytes = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
name = 'supplier_%s_%s' % (
nameurl.GetName(self.supplier_idurl),
diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
)
self.request_packet_id = None
self.callbacks = {}
try:
st = bpio.ReadTextFile(
settings.SupplierServiceFilename(
idurl=self.supplier_idurl,
customer_idurl=self.customer_idurl,
)).strip()
except:
st = 'DISCONNECTED'
automat.Automat.__init__(
self,
name,
state=st,
debug_level=_DebugLevel,
log_events=_Debug,
log_transitions=_Debug,
)
for cb in self.callbacks.values():
cb(self.supplier_idurl, self.state, self.state)
def _on_my_dht_relations_discovered(self, dht_result):
if not (dht_result and isinstance(dht_result, dict)
and len(dht_result.get('suppliers', [])) > 0):
lg.warn('no dht records found for my customer family')
self.automat('suppliers-read-failed')
return
dht_suppliers = id_url.to_bin_list(dht_result['suppliers'])
dht_ecc_map = dht_result.get('ecc_map', settings.DefaultEccMapName())
try:
dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber(
dht_ecc_map)
except:
lg.exc()
dht_desired_suppliers_number = eccmap.GetEccMapSuppliersNumber(
settings.DefaultEccMapName())
settings.config.conf().setInt('services/customer/suppliers-number',
dht_desired_suppliers_number)
contactsdb.set_suppliers(dht_suppliers)
contactsdb.save_suppliers()
lg.info('found and restored list of %d suppliers from DHT' %
dht_desired_suppliers_number)
self.automat('suppliers-read-ok')
def do_calculate_needed_bytes(self):
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
num_suppliers = -1
if self.customer_idurl == my_id.getLocalIDURL():
num_suppliers = settings.getSuppliersNumberDesired()
else:
known_ecc_map = contactsdb.get_customer_meta_info(self.customer_idurl).get('ecc_map')
if known_ecc_map:
num_suppliers = eccmap.GetEccMapSuppliersNumber(known_ecc_map)
if num_suppliers > 0:
self.needed_bytes = int(math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
raise Exception('not possible to determine needed_bytes value to be requested from that supplier')
def _do_merge_revisions(self, dht_info, my_info, latest_revision):
if dht_info is None or not isinstance(dht_info, dict):
merged_info = my_info
else:
if latest_revision == int(dht_info['revision']):
if my_info is not None:
if latest_revision == int(my_info['revision']):
# I have same revision as info from DHT
merged_info = dht_info
else:
if int(my_info['revision']) > int(
dht_info['revision']):
# here my revision is higher, so I have some changes that needs to be published already
merged_info = my_info
else:
# here my revision is lower, I need to take info from DHT
merged_info = dht_info
else:
merged_info = dht_info
else:
# here my revision is higher, so I have some changes that needs to be published already
merged_info = my_info
if not merged_info:
return None
# make sure list of suppliers have correct length according to ecc_map
if not merged_info['ecc_map']:
known_ecc_map = contactsdb.get_customer_meta_info(
self.customer_idurl).get('ecc_map', None)
lg.warn('unknown ecc_map, will populate known value: %s' %
known_ecc_map)
merged_info['ecc_map'] = known_ecc_map
if merged_info['ecc_map']:
expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
merged_info['ecc_map'])
if len(merged_info['suppliers']) < expected_suppliers_count:
merged_info['suppliers'] += [
b'',
] * (expected_suppliers_count - len(merged_info['suppliers']))
elif len(merged_info['suppliers']) > expected_suppliers_count:
merged_info['suppliers'] = merged_info[
'suppliers'][:expected_suppliers_count]
if merged_info['revision'] != latest_revision:
lg.info('will switch known revision %d to the latest: %d' % (
merged_info['revision'],
latest_revision,
))
merged_info['revision'] = latest_revision
return merged_info
def doCheckAllConnected(self, *args, **kwargs):
"""
Action method.
"""
connected_count = 0
for supplier_idurl in self.known_suppliers_list:
if not id_url.is_cached(supplier_idurl):
continue
sc = supplier_connector.by_idurl(
supplier_idurl, customer_idurl=self.customer_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
connected_count += 1
critical_suppliers_number = 1
if self.known_ecc_map:
from raid import eccmap
critical_suppliers_number = eccmap.GetCorrectableErrors(
eccmap.GetEccMapSuppliersNumber(self.known_ecc_map))
if connected_count >= critical_suppliers_number:
self.automat('all-suppliers-connected')
def _do_check_supplier_connectors(self):
connected_count = 0
for supplier_idurl in self.known_suppliers_list:
if not id_url.is_cached(supplier_idurl):
continue
sc = supplier_connector.by_idurl(
supplier_idurl, customer_idurl=self.customer_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
connected_count += 1
critical_suppliers_number = 1
if self.known_ecc_map:
from raid import eccmap
critical_suppliers_number = eccmap.GetCorrectableErrors(
eccmap.GetEccMapSuppliersNumber(self.known_ecc_map))
if _Debug:
lg.args(_DebugLevel,
connected_count=connected_count,
critical_suppliers_number=critical_suppliers_number)
if connected_count >= critical_suppliers_number:
self.automat('all-suppliers-connected')
def doRebuildFamily(self, *args, **kwargs):
"""
Action method.
"""
dht_info_valid = self._do_validate_dht_info(args[0])
my_info_valid = self._do_validate_my_info(self.my_info)
latest_revision = self._do_detect_latest_revision(
dht_info_valid, my_info_valid)
merged_info = None
if latest_revision > 0:
merged_info = self._do_merge_revisions(dht_info_valid,
my_info_valid,
latest_revision)
if not merged_info:
merged_info = self._do_create_first_revision(self.current_request)
# if not merged_info:
# lg.err('failed to merge customer family info after reading from DHT, skip transaction')
# self.transaction = None
# return
possible_transaction = self._do_process_request(
merged_info, self.current_request)
if not possible_transaction:
lg.err(
'failed to process customer family change request, skip transaction'
)
return
self.transaction = self._do_increment_revision(possible_transaction)
if _Debug:
lg.out(
_DebugLevel,
'family_member._do_build_transaction : %r' % self.transaction)
self.refresh_period = DHT_RECORD_REFRESH_INTERVAL * settings.DefaultDesiredSuppliers(
)
if self.transaction:
known_ecc_map = self.transaction.get('ecc_map')
if known_ecc_map:
expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
known_ecc_map)
self.refresh_period = DHT_RECORD_REFRESH_INTERVAL * expected_suppliers_count
def _do_process_family_refresh_request(self, merged_info):
if not self.my_info:
self.my_info = self._do_create_possible_revision(
int(merged_info['revision']))
lg.warn(
'"family-refresh" request will use "possible" customer meta info: %r'
% self.my_info)
if int(self.my_info['revision']) > int(merged_info['revision']):
lg.info(
'"family-refresh" request will overwrite DHT record with my info because my revision is higher than record in DHT'
)
return self.my_info.copy()
try:
my_position = self.my_info['suppliers'].index(
my_id.getLocalIDURL())
except:
my_position = -1
if my_position < 0:
lg.warn(
'"family-refresh" request failed because my info not exist or not valid, my own position in the family is unknown'
)
return None
my_expected_suppliers_count = None
if self.my_info['ecc_map']:
my_expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
self.my_info['ecc_map'])
if my_expected_suppliers_count and my_position >= my_expected_suppliers_count:
lg.warn(
'"family-refresh" request failed because my info is not valid, supplier position greater than expected suppliers count'
)
return None
if len(merged_info['suppliers']) != my_expected_suppliers_count:
lg.warn(
'number of suppliers not expected during processing of "family-refresh" request'
)
if len(merged_info['suppliers']) < my_expected_suppliers_count:
merged_info['suppliers'] += [
b'',
] * (my_expected_suppliers_count -
len(merged_info['suppliers']))
else:
merged_info['suppliers'] = merged_info[
'suppliers'][:my_expected_suppliers_count]
try:
existing_position = merged_info['suppliers'].index(
my_id.getLocalIDURL())
except ValueError:
existing_position = -1
if existing_position < 0:
if merged_info['suppliers'][my_position] not in [b'', '', None]:
# TODO: SECURITY need to implement a signature verification and
# also build solution to validate that change was approved by customer
lg.warn(
'overwriting another supplier %s with my IDURL at position %d in family of customer %s'
% (
merged_info['suppliers'][my_position],
my_position,
self.customer_idurl,
))
merged_info['suppliers'][my_position] = my_id.getLocalIDURL()
if _Debug:
lg.out(
_DebugLevel,
' placed supplier %s at known position %d in the family of customer %s'
%
(my_id.getLocalIDURL(), my_position, self.customer_idurl))
existing_position = my_position
if existing_position != my_position:
merged_info['suppliers'][existing_position] = b''
merged_info['suppliers'][my_position] = my_id.getLocalIDURL()
if _Debug:
lg.out(
_DebugLevel,
' found my IDURL on %d position and will move it on %d position in the family of customer %s'
% (existing_position, my_position, self.customer_idurl))
return merged_info
def _do_process_family_join_request(self, merged_info, current_request):
current_request_expected_suppliers_count = None
if current_request['ecc_map']:
current_request_expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
current_request['ecc_map'])
if current_request_expected_suppliers_count and current_request[
'position'] > current_request_expected_suppliers_count:
lg.warn(
'"family-join" request is not valid, supplier position %d greater than expected suppliers count %d for %s'
% (current_request['position'],
current_request_expected_suppliers_count,
current_request['ecc_map']))
return None
if merged_info['ecc_map'] and current_request[
'ecc_map'] and current_request['ecc_map'] != merged_info[
'ecc_map']:
lg.info(
'from "family-join" request, detected ecc_map change %s -> %s for customer %s'
% (merged_info['ecc_map'], current_request['ecc_map'],
self.customer_idurl))
merged_info['ecc_map'] = current_request['ecc_map']
if not merged_info['ecc_map'] and current_request['ecc_map']:
lg.info(
'from "family-join" request, detected ecc_map was set to %s for the first time for customer %s'
% (current_request['ecc_map'], self.customer_idurl))
merged_info['ecc_map'] = current_request['ecc_map']
if not merged_info['ecc_map']:
lg.warn(
'still did not found actual ecc_map from DHT or from the request'
)
return None
expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
merged_info['ecc_map'])
if not merged_info['suppliers']:
merged_info['suppliers'] = [
b'',
] * expected_suppliers_count
if len(merged_info['suppliers']) < expected_suppliers_count:
merged_info['suppliers'] += [
b'',
] * (expected_suppliers_count - len(merged_info['suppliers']))
else:
merged_info['suppliers'] = merged_info[
'suppliers'][:expected_suppliers_count]
try:
existing_position = merged_info['suppliers'].index(
current_request['supplier_idurl'])
except:
existing_position = -1
if current_request[
'position'] is not None and current_request['position'] >= 0:
if current_request['position'] >= expected_suppliers_count:
lg.warn(
'"family-join" request is not valid, supplier position greater than expected suppliers count'
)
return None
if existing_position >= 0 and existing_position != current_request[
'position']:
merged_info['suppliers'][existing_position] = b''
merged_info['suppliers'][current_request[
'position']] = current_request['supplier_idurl']
if _Debug:
lg.out(
_DebugLevel,
' found my IDURL on %d position and will move it on %d position in the family of customer %s'
% (existing_position, current_request['position'],
self.customer_idurl))
if merged_info['suppliers'][current_request[
'position']] != current_request['supplier_idurl']:
if merged_info['suppliers'][
current_request['position']] not in [b'', '', None]:
# TODO: SECURITY need to implement a signature verification and
# also build solution to validate that change was approved by customer
lg.warn(
'overwriting another supplier %s with my IDURL at position %d in family of customer %s'
% (
merged_info['suppliers'][
current_request['position']],
current_request['position'],
self.customer_idurl,
))
merged_info['suppliers'][current_request[
'position']] = current_request['supplier_idurl']
if _Debug:
lg.out(
_DebugLevel,
' placed supplier %s at known position %d in the family of customer %s'
% (current_request['supplier_idurl'],
current_request['position'], self.customer_idurl))
if current_request['supplier_idurl'] not in merged_info['suppliers']:
if b'' in merged_info['suppliers']:
first_empty_position = merged_info['suppliers'].index(b'')
merged_info['suppliers'][
first_empty_position] = current_request['supplier_idurl']
if _Debug:
lg.out(
_DebugLevel,
' placed supplier %s at first empty position %d in family of customer %s'
% (current_request['supplier_idurl'],
first_empty_position, self.customer_idurl))
else:
merged_info['suppliers'].append(
current_request['supplier_idurl'])
if _Debug:
lg.out(
_DebugLevel,
' added supplier %s to family of customer %s' %
(current_request['supplier_idurl'],
self.customer_idurl))
return merged_info