def isMoreNeeded(self, *args, **kwargs):
"""
Condition method.
"""
# if _Debug:
# lg.out(_DebugLevel, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
# contactsdb.num_suppliers(), len(self.dismiss_list),
# settings.getSuppliersNumberDesired()))
if id_url.is_some_empty(contactsdb.suppliers()):
if _Debug:
lg.out(_DebugLevel,
'fire_hire.isMoreNeeded found empty supplier!!!')
return True
if isinstance(args[0], list):
dismissed = args[0]
else:
dismissed = self.dismiss_list
s = set(id_url.to_bin_list(contactsdb.suppliers()))
s.difference_update(set(id_url.to_bin_list(dismissed)))
result = len(s) < settings.getSuppliersNumberDesired()
if _Debug:
lg.out(
_DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' %
(contactsdb.num_suppliers(), len(dismissed), len(s),
settings.getSuppliersNumberDesired(), result))
return result
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(10,
'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
result = list(result)
lg.out(10, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def isMoreNeeded(self, arg):
"""
Condition method.
"""
# lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
# contactsdb.num_suppliers(), len(self.dismiss_list),
# settings.getSuppliersNumberDesired()))
if '' in contactsdb.suppliers():
if _Debug:
lg.out(_DebugLevel,
'fire_hire.isMoreNeeded found empty suppliers!!!')
return True
if isinstance(arg, list):
dismissed = arg
else:
dismissed = self.dismiss_list
s = set(contactsdb.suppliers())
s.difference_update(set(dismissed))
result = len(s) < settings.getSuppliersNumberDesired()
if _Debug:
lg.out(
_DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' %
(contactsdb.num_suppliers(), len(dismissed), len(s),
settings.getSuppliersNumberDesired(), result))
return result
def isStillNeeded(self, *args, **kwargs):
"""
Condition method.
"""
supplier_idurl = args[0]
current_suppliers = contactsdb.suppliers()
if supplier_idurl in current_suppliers:
# this guy is already a supplier, we still need more then
return True
desired_number = settings.getSuppliersNumberDesired()
needed_suppliers = current_suppliers[:desired_number]
empty_suppliers = needed_suppliers.count(id_url.field(b''))
# if '' in needed_suppliers:
# lg.warn('found empty suppliers!!!')
# return True
s = set(id_url.to_bin_list(needed_suppliers))
s.add(id_url.to_bin(supplier_idurl))
s.difference_update(set(id_url.to_bin_list(self.dismiss_list)))
result = len(s) - empty_suppliers < settings.getSuppliersNumberDesired(
)
# if _Debug:
# lg.out(_DebugLevel, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % (
# contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list),
# len(s), settings.getSuppliersNumberDesired(), result))
return result
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(
settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
result = list(result)
lg.out(10, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def doFindNewSupplier(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r'
%
(settings.getSuppliersNumberDesired(), contactsdb.suppliers()))
from p2p import network_connector
if network_connector.A().state != 'CONNECTED':
if _Debug:
lg.out(
_DebugLevel,
' network_connector is not CONNECTED at the moment, SKIP'
)
self.automat('search-failed')
return
position_for_new_supplier = None
for pos in range(settings.getSuppliersNumberDesired()):
if pos in self.hire_list:
continue
supplier_idurl = contactsdb.supplier(pos)
if not supplier_idurl:
lg.info(
'found empty supplier at position %d and going to find new supplier on that position'
% pos)
position_for_new_supplier = pos
break
if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False):
lg.info(
'going to find new supplier on existing position %d to replace supplier %s'
% (
pos,
supplier_idurl,
))
position_for_new_supplier = pos
break
if position_for_new_supplier is None:
lg.err('did not found position for new supplier')
self.automat('search-failed')
return
from customer import supplier_finder
for idurl_txt in strng.to_text(config.conf().getData(
'services/employer/candidates')).split(','):
if idurl_txt.strip():
supplier_finder.AddSupplierToHire(idurl_txt)
self.hire_list.append(position_for_new_supplier)
supplier_finder.A(
'start',
family_position=position_for_new_supplier,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(contactsdb.suppliers()),
)
def doRemoveSuppliers(self, arg):
"""
Action method.
"""
current_suppliers = contactsdb.suppliers()
desired_suppliers = settings.getSuppliersNumberDesired()
if len(current_suppliers) < desired_suppliers:
lg.warn('must have more suppliers %d<%d' % (
len(current_suppliers), desired_suppliers))
for supplier_idurl in self.dismiss_list:
if supplier_idurl not in current_suppliers:
lg.warn('%s not a supplier' % supplier_idurl)
continue
pos = current_suppliers.index(supplier_idurl)
# current_suppliers.remove(supplier_idurl)
current_suppliers[pos] = ''
misc.writeSupplierData(
supplier_idurl,
'disconnected',
time.strftime('%d-%m-%Y %H:%M:%S'))
current_suppliers = current_suppliers[:desired_suppliers]
contactsdb.update_suppliers(current_suppliers)
contactsdb.save_suppliers()
if settings.NewWebGUI():
from web import control
control.on_suppliers_changed(current_suppliers)
else:
from web import webcontrol
webcontrol.OnListSuppliers()
lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def doFindNewSupplier(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'fire_hire.doFindNewSupplier')
position_for_new_supplier = None
for pos in range(settings.getSuppliersNumberDesired()):
if pos in self.hire_list:
continue
supplier_idurl = contactsdb.supplier(pos)
if not supplier_idurl:
lg.info('found empty supplier at position %d and going to find new supplier on that position' % pos)
position_for_new_supplier = pos
break
if supplier_idurl in self.dismiss_list:
lg.info('going to find new supplier on existing position %d to replace supplier %s' % (
pos, supplier_idurl, ))
position_for_new_supplier = pos
break
if position_for_new_supplier is None:
lg.err('did not found position for new supplier')
self.automat('search-failed')
return
self.hire_list.append(position_for_new_supplier)
supplier_finder.A(
'start',
family_position=position_for_new_supplier,
ecc_map=eccmap.Current().name,
family_snapshot=contactsdb.suppliers(),
)
def doRemoveSuppliers(self, *args, **kwargs):
"""
Action method.
"""
current_suppliers = contactsdb.suppliers()
desired_suppliers = settings.getSuppliersNumberDesired()
if len(current_suppliers) < desired_suppliers:
lg.warn('must have more suppliers %d<%d' % (
len(current_suppliers), desired_suppliers))
removed_suppliers = []
for supplier_idurl in self.dismiss_list:
if supplier_idurl not in current_suppliers:
lg.warn('%s not a supplier' % supplier_idurl)
continue
pos = current_suppliers.index(supplier_idurl)
# current_suppliers.remove(supplier_idurl)
current_suppliers[pos] = ''
removed_suppliers.append((pos, supplier_idurl,))
misc.writeSupplierData(
supplier_idurl,
'disconnected',
time.strftime('%d-%m-%Y %H:%M:%S'),
my_id.getLocalID(),
)
current_suppliers = current_suppliers[:desired_suppliers]
contactsdb.update_suppliers(current_suppliers)
contactsdb.save_suppliers()
from main import control
control.on_suppliers_changed(current_suppliers)
for position, supplier_idurl in removed_suppliers:
events.send('supplier-modified', dict(
new_idurl=None, old_idurl=supplier_idurl, position=position,
))
lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def doSaveConfig(self, arg):
"""
Action method.
"""
self.configs = (settings.getSuppliersNumberDesired(),
diskspace.GetBytesFromString(
settings.getNeededString()))
def _stats(params):
from contacts import contactsdb
from p2p import contact_status
from lib import diskspace
result = {}
result['suppliers'] = contactsdb.num_suppliers()
result['max_suppliers'] = settings.getSuppliersNumberDesired()
result['online_suppliers'] = contact_status.countOnlineAmong(
contactsdb.suppliers())
result['customers'] = contactsdb.num_customers()
result['bytes_donated'] = settings.getDonatedBytes()
result['value_donated'] = diskspace.MakeStringFromBytes(
settings.getDonatedBytes())
result['bytes_needed'] = settings.getNeededBytes()
result['value_needed'] = diskspace.MakeStringFromBytes(
settings.getNeededBytes())
result['bytes_used_total'] = backup_fs.sizebackups()
result['value_used_total'] = diskspace.MakeStringFromBytes(
backup_fs.sizebackups())
result['bytes_used_supplier'] = 0 if (
contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() /
contactsdb.num_suppliers()))
result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
result['files_count'] = backup_fs.numberfiles()
result['folders_count'] = backup_fs.numberfolders()
result['items_count'] = backup_fs.counter()
result['timestamp'] = time.time()
return {
'result': result,
}
def doSaveConfig(self, arg):
"""
Action method.
"""
self.configs = (
settings.getSuppliersNumberDesired(),
diskspace.GetBytesFromString(
settings.getNeededString()))
def CurrentName():
"""
Should return a ecc map name from current suppliers number - taken from user settings.
"""
from main import settings
snum = settings.getSuppliersNumberDesired()
if snum < 0:
return DefaultName()
return GetEccMapName(snum)
def isConfigChanged(self, *args, **kwargs):
"""
Condition method.
"""
curconfigs = (settings.getSuppliersNumberDesired(),
diskspace.GetBytesFromString(settings.getNeededString()))
if None in self.configs:
return True
return self.configs[0] != curconfigs[0] or self.configs[1] != curconfigs[1]
def CurrentName():
"""
Should return a ecc map name from current suppliers number - taken from user settings.
"""
from main import settings
snum = settings.getSuppliersNumberDesired()
if snum < 0:
return DefaultName()
return GetEccMapName(snum)
def __init__(self,
supplier_idurl,
customer_idurl,
needed_bytes,
key_id=None,
queue_subscribe=True):
"""
"""
self.supplier_idurl = supplier_idurl
self.customer_idurl = customer_idurl
self.needed_bytes = needed_bytes
self.key_id = key_id
self.queue_subscribe = queue_subscribe
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(
settings.getNeededString(), 0)
num_suppliers = -1
if self.customer_idurl == my_id.getLocalIDURL():
num_suppliers = settings.getSuppliersNumberDesired()
else:
known_ecc_map = contactsdb.get_customer_meta_info(
customer_idurl).get('ecc_map')
if known_ecc_map:
num_suppliers = eccmap.GetEccMapSuppliersNumber(
known_ecc_map)
if num_suppliers > 0:
self.needed_bytes = int(
math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
raise Exception(
'not possible to determine needed_bytes value to be requested from that supplier'
)
# self.needed_bytes = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
name = 'supplier_%s_%s' % (
nameurl.GetName(self.supplier_idurl),
diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
)
self.request_packet_id = None
self.callbacks = {}
try:
st = bpio.ReadTextFile(
settings.SupplierServiceFilename(
idurl=self.supplier_idurl,
customer_idurl=self.customer_idurl,
)).strip()
except:
st = 'DISCONNECTED'
automat.Automat.__init__(
self,
name,
state=st,
debug_level=_DebugLevel,
log_events=_Debug,
log_transitions=_Debug,
)
for cb in self.callbacks.values():
cb(self.supplier_idurl, self.state, self.state)
def IsAllHired():
"""
"""
if settings.getSuppliersNumberDesired() < 0:
# I must know how many suppliers I want
lg.warn('my desired number of suppliers not set')
return False
if contactsdb.num_suppliers() != settings.getSuppliersNumberDesired():
# I must have exactly that amount of suppliers already
if _Debug:
lg.args(_DebugLevel, desiried_suppliers=settings.getSuppliersNumberDesired(),
current_suppliers=contactsdb.num_suppliers(), )
return False
if id_url.is_some_empty(contactsdb.suppliers()):
# I must know all of my suppliers
if _Debug:
lg.args(_DebugLevel, my_suppliers=contactsdb.suppliers())
return False
return True
def isConfigChanged(self, arg):
"""
Condition method.
"""
curconfigs = (settings.getSuppliersNumberDesired(),
diskspace.GetBytesFromString(settings.getNeededString()))
if None in self.configs:
return True
return self.configs[0] != curconfigs[
0] or self.configs[1] != curconfigs[1]
def isMoreNeeded(self, arg):
"""
Condition method.
"""
# lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
# contactsdb.num_suppliers(), len(self.dismiss_list),
# settings.getSuppliersNumberDesired()))
if '' in contactsdb.suppliers():
lg.out(4, 'fire_hire.isMoreNeeded found empty suppliers!!!')
return True
if isinstance(arg, list):
dismissed = arg
else:
dismissed = self.dismiss_list
s = set(contactsdb.suppliers())
s.difference_update(set(dismissed))
result = len(s) < settings.getSuppliersNumberDesired()
lg.out(14, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % (
contactsdb.num_suppliers(), len(dismissed), len(s),
settings.getSuppliersNumberDesired(), result))
return result
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
disconnected_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(6, 'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
elif sc.state == 'DISCONNECTED':
disconnected_suppliers.add(supplier_idurl)
if contact_status.isOffline(supplier_idurl):
disconnected_suppliers.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(
settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
if disconnected_suppliers:
from raid import eccmap
if len(disconnected_suppliers) >= eccmap.GetFireHireErrors(settings.getSuppliersNumberDesired()):
result.update(disconnected_suppliers)
result = list(result)
lg.out(6, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def do_calculate_needed_bytes(self):
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
num_suppliers = -1
if self.customer_idurl == my_id.getLocalIDURL():
num_suppliers = settings.getSuppliersNumberDesired()
else:
known_ecc_map = contactsdb.get_customer_meta_info(self.customer_idurl).get('ecc_map')
if known_ecc_map:
num_suppliers = eccmap.GetEccMapSuppliersNumber(known_ecc_map)
if num_suppliers > 0:
self.needed_bytes = int(math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
raise Exception('not possible to determine needed_bytes value to be requested from that supplier')
def isStillNeeded(self, arg):
"""
Condition method.
"""
supplier_idurl = arg
current_suppliers = contactsdb.suppliers()
if supplier_idurl in current_suppliers:
# this guy is already a supplier, we still need more then
return True
desired_number = settings.getSuppliersNumberDesired()
needed_suppliers = current_suppliers[:desired_number]
empty_suppliers = needed_suppliers.count('')
# if '' in needed_suppliers:
# lg.warn('found empty suppliers!!!')
# return True
s = set(needed_suppliers)
s.add(supplier_idurl)
s.difference_update(set(self.dismiss_list))
result = len(s) - \
empty_suppliers < settings.getSuppliersNumberDesired()
# lg.out(14, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % (
# contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list),
# len(s), settings.getSuppliersNumberDesired(), result))
return result
def doRemoveSuppliers(self, *args, **kwargs):
"""
Action method.
"""
current_suppliers = contactsdb.suppliers()
desired_suppliers = settings.getSuppliersNumberDesired()
if len(current_suppliers) < desired_suppliers:
lg.warn('must have more suppliers %d<%d' %
(len(current_suppliers), desired_suppliers))
removed_suppliers = []
for supplier_idurl in self.dismiss_list:
if id_url.is_not_in(supplier_idurl,
current_suppliers,
as_field=False):
lg.warn('%s not a supplier' % supplier_idurl)
continue
pos = current_suppliers.index(id_url.field(supplier_idurl))
current_suppliers[pos] = ''
removed_suppliers.append((
pos,
supplier_idurl,
))
misc.writeSupplierData(
supplier_idurl,
'disconnected',
time.strftime('%d-%m-%Y %H:%M:%S'),
my_id.getLocalID(),
)
current_suppliers = current_suppliers[:desired_suppliers]
contactsdb.update_suppliers(current_suppliers)
contactsdb.save_suppliers()
from main import control
control.on_suppliers_changed(current_suppliers)
for position, supplier_idurl in removed_suppliers:
events.send(
'supplier-modified',
dict(
new_idurl=None,
old_idurl=supplier_idurl,
position=position,
))
lg.info(
'removed some suppliers : %d desired_suppliers=%d current_suppliers=%d'
% (len(self.dismiss_list), desired_suppliers,
len(contactsdb.suppliers())))
if _Debug:
lg.out(_DebugLevel,
' my current suppliers: %r' % contactsdb.suppliers())
def __init__(self,
supplier_idurl,
customer_idurl,
needed_bytes,
key_id=None,
queue_subscribe=True):
"""
"""
self.supplier_idurl = supplier_idurl
self.customer_idurl = customer_idurl
self.needed_bytes = needed_bytes
self.key_id = key_id
self.queue_subscribe = queue_subscribe
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(
settings.getNeededString(), 0)
num_suppliers = settings.getSuppliersNumberDesired()
if num_suppliers > 0:
self.needed_bytes = int(
math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
self.needed_bytes = int(
math.ceil(2.0 * settings.MinimumNeededBytes() /
float(settings.DefaultDesiredSuppliers())))
name = 'supplier_%s_%s' % (
nameurl.GetName(self.supplier_idurl),
diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
)
self.request_packet_id = None
self.callbacks = {}
try:
st = bpio.ReadTextFile(
settings.SupplierServiceFilename(
idurl=self.supplier_idurl,
customer_idurl=self.customer_idurl,
)).strip()
except:
st = 'DISCONNECTED'
automat.Automat.__init__(
self,
name,
state=st,
debug_level=_DebugLevel,
log_events=_Debug,
log_transitions=_Debug,
)
for cb in self.callbacks.values():
cb(self.supplier_idurl, self.state, self.state)
def doRequestService(self, arg):
"""
Action method.
"""
bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(), 0)
num_suppliers = settings.getSuppliersNumberDesired()
if num_suppliers > 0:
bytes_per_supplier = int(math.ceil(2.0 * bytes_needed / float(num_suppliers)))
else:
bytes_per_supplier = int(math.ceil(2.0 * settings.MinimumNeededBytes() / float(settings.DefaultDesiredSuppliers())))
service_info = 'service_supplier %d' % bytes_per_supplier
request = p2p_service.SendRequestService(
self.idurl, service_info, callbacks={
commands.Ack(): self._supplier_acked,
commands.Fail(): self._supplier_failed})
# commands.Ack(): lambda response, info: self.automat('ack', response),
# commands.Fail(): lambda response, info: self.automat('fail', response)})
self.request_packet_id = request.PacketID
def __init__(self, supplier_idurl, customer_idurl, needed_bytes):
"""
"""
self.supplier_idurl = supplier_idurl
self.customer_idurl = customer_idurl
self.needed_bytes = needed_bytes
if self.needed_bytes is None:
total_bytes_needed = diskspace.GetBytesFromString(
settings.getNeededString(), 0)
num_suppliers = settings.getSuppliersNumberDesired()
if num_suppliers > 0:
self.needed_bytes = int(
math.ceil(2.0 * total_bytes_needed / float(num_suppliers)))
else:
self.needed_bytes = int(
math.ceil(2.0 * settings.MinimumNeededBytes() /
float(settings.DefaultDesiredSuppliers())))
name = 'supplier_%s_%s' % (
nameurl.GetName(self.supplier_idurl),
diskspace.MakeStringFromBytes(self.needed_bytes).replace(' ', ''),
)
self.request_packet_id = None
self.callbacks = {}
try:
st = bpio.ReadTextFile(
settings.SupplierServiceFilename(
idurl=self.supplier_idurl,
customer_idurl=self.customer_idurl,
)).strip()
except:
st = 'DISCONNECTED'
if st == 'CONNECTED':
automat.Automat.__init__(self, name, 'CONNECTED', _DebugLevel,
_Debug)
elif st == 'NO_SERVICE':
automat.Automat.__init__(self, name, 'NO_SERVICE', _DebugLevel,
_Debug)
else:
automat.Automat.__init__(self, name, 'DISCONNECTED', _DebugLevel,
_Debug)
for cb in self.callbacks.values():
cb(self.supplier_idurl, self.state, self.state)
def _stats(params):
from contacts import contactsdb
from p2p import contact_status
from lib import diskspace
result = {}
result['suppliers'] = contactsdb.num_suppliers()
result['max_suppliers'] = settings.getSuppliersNumberDesired()
result['online_suppliers'] = contact_status.countOnlineAmong(contactsdb.suppliers())
result['customers'] = contactsdb.num_customers()
result['bytes_donated'] = settings.getDonatedBytes()
result['value_donated'] = diskspace.MakeStringFromBytes(settings.getDonatedBytes())
result['bytes_needed'] = settings.getNeededBytes()
result['value_needed'] = diskspace.MakeStringFromBytes(settings.getNeededBytes())
result['bytes_used_total'] = backup_fs.sizebackups()
result['value_used_total'] = diskspace.MakeStringFromBytes(backup_fs.sizebackups())
result['bytes_used_supplier'] = 0 if (contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers()))
result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
result['files_count'] = backup_fs.numberfiles()
result['folders_count'] = backup_fs.numberfolders()
result['items_count'] = backup_fs.counter()
result['timestamp'] = time.time()
return {'result': result, }
def doRequestService(self, arg):
"""
Action method.
"""
bytes_needed = diskspace.GetBytesFromString(settings.getNeededString(),
0)
num_suppliers = settings.getSuppliersNumberDesired()
if num_suppliers > 0:
bytes_per_supplier = int(
math.ceil(2.0 * bytes_needed / float(num_suppliers)))
else:
bytes_per_supplier = int(
math.ceil(2.0 * settings.MinimumNeededBytes() /
float(settings.DefaultDesiredSuppliers())))
service_info = 'service_supplier %d' % bytes_per_supplier
request = p2p_service.SendRequestService(self.idurl,
service_info,
callbacks={
commands.Ack():
self._supplier_acked,
commands.Fail():
self._supplier_failed,
})
self.request_packet_id = request.PacketID
def doDecideToDismiss(self, *args, **kwargs):
"""
Action method.
"""
global _SuppliersToFire
from p2p import p2p_connector
from p2p import network_connector
from customer import supplier_connector
from p2p import online_status
# take any actions only if I am connected to the network
if not p2p_connector.A() or not network_connector.A():
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss p2p_connector() is not ready yet, SKIP'
)
self.automat('made-decision', [])
return
if not network_connector.A():
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss network_connector() is not ready yet, SKIP'
)
self.automat('made-decision', [])
return
if p2p_connector.A().state != 'CONNECTED' or network_connector.A(
).state != 'CONNECTED':
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss p2p/network is not connected at the moment, SKIP'
)
self.automat('made-decision', [])
return
# if certain suppliers needs to be removed by manual/external request just do that
to_be_fired = id_url.to_list(set(_SuppliersToFire))
_SuppliersToFire = []
if to_be_fired:
lg.info('going to fire %d suppliers from external request' %
len(to_be_fired))
self.automat('made-decision', to_be_fired)
return
# make sure to not go too far when i just want to decrease number of my suppliers
number_desired = settings.getSuppliersNumberDesired()
redundant_suppliers = set()
if contactsdb.num_suppliers() > number_desired:
for supplier_index in range(number_desired,
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
lg.info('found REDUNDANT supplier %s at position %d' % (
idurl,
supplier_index,
))
redundant_suppliers.add(idurl)
if redundant_suppliers:
result = list(redundant_suppliers)
lg.info('will replace redundant suppliers: %s' % result)
self.automat('made-decision', result)
return
# now I need to look more careful at my suppliers
potentialy_fired = set()
connected_suppliers = set()
disconnected_suppliers = set()
requested_suppliers = set()
online_suppliers = set()
offline_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
my_suppliers = contactsdb.suppliers()
if _Debug:
lg.args(_DebugLevel, my_suppliers=my_suppliers)
if id_url.is_some_empty(my_suppliers):
lg.warn('SKIP, found empty supplier')
self.automat('made-decision', [])
return
for supplier_idurl in my_suppliers:
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
lg.warn('SKIP, supplier connector for supplier %s not exist' %
supplier_idurl)
continue
if sc.state == 'NO_SERVICE':
lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
disconnected_suppliers.add(supplier_idurl)
potentialy_fired.add(supplier_idurl)
elif sc.state == 'CONNECTED':
connected_suppliers.add(supplier_idurl)
elif sc.state in [
'DISCONNECTED',
'REFUSE',
]:
disconnected_suppliers.add(supplier_idurl)
# elif sc.state in ['QUEUE?', 'REQUEST', ]:
# requested_suppliers.add(supplier_idurl)
if online_status.isOffline(supplier_idurl):
offline_suppliers.add(supplier_idurl)
elif online_status.isOnline(supplier_idurl):
online_suppliers.add(supplier_idurl)
elif online_status.isCheckingNow(supplier_idurl):
requested_suppliers.add(supplier_idurl)
if not connected_suppliers or not online_suppliers:
lg.warn('SKIP, no ONLINE suppliers found at the moment')
self.automat('made-decision', [])
return
if requested_suppliers:
lg.warn('SKIP, still waiting response from some of suppliers')
self.automat('made-decision', [])
return
if not disconnected_suppliers:
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss SKIP, no OFFLINE suppliers found at the moment'
)
# TODO: add more conditions to fire "slow" suppliers - they are still connected but useless
self.automat('made-decision', [])
return
if len(offline_suppliers) + len(online_suppliers) != number_desired:
lg.warn('SKIP, offline + online != total count: %s %s %s' %
(offline_suppliers, online_suppliers, number_desired))
self.automat('made-decision', [])
return
max_offline_suppliers_count = eccmap.GetCorrectableErrors(
number_desired)
if len(offline_suppliers) > max_offline_suppliers_count:
lg.warn(
'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
len(offline_suppliers),
max_offline_suppliers_count,
))
self.automat('made-decision', [])
return
critical_offline_suppliers_count = eccmap.GetFireHireErrors(
number_desired)
if len(offline_suppliers) >= critical_offline_suppliers_count and len(
offline_suppliers) > 0:
if config.conf().getBool(
'services/employer/replace-critically-offline-enabled'):
# TODO: check that issue
# too aggressive replacing suppliers who still have the data is very dangerous !!!
one_dead_supplier = offline_suppliers.pop()
lg.warn(
'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
% (
one_dead_supplier,
critical_offline_suppliers_count,
))
potentialy_fired.add(one_dead_supplier)
if not potentialy_fired:
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!'
)
self.automat('made-decision', [])
return
# only replace suppliers one by one at the moment
result = list(potentialy_fired)
lg.info('will replace supplier %s' % result[0])
self.automat('made-decision', [
result[0],
])
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
to_be_fired = list(set(_SuppliersToFire))
_SuppliersToFire = []
if to_be_fired:
lg.warn('going to fire %d suppliers from external request' %
len(to_be_fired))
self.automat('made-decision', to_be_fired)
return
potentialy_fired = set()
connected_suppliers = set()
disconnected_suppliers = set()
requested_suppliers = set()
online_suppliers = set()
offline_suppliers = set()
redundant_suppliers = set()
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers() or None in contactsdb.suppliers():
lg.warn('SKIP, found empty supplier')
self.automat('made-decision', [])
return
number_desired = settings.getSuppliersNumberDesired()
for supplier_idurl in contactsdb.suppliers():
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
lg.warn('SKIP, supplier connector for supplier %s not exist' %
supplier_idurl)
continue
if sc.state == 'NO_SERVICE':
lg.warn('found "NO_SERVICE" supplier: %s' % supplier_idurl)
disconnected_suppliers.add(supplier_idurl)
potentialy_fired.add(supplier_idurl)
elif sc.state == 'CONNECTED':
connected_suppliers.add(supplier_idurl)
elif sc.state in [
'DISCONNECTED',
'REFUSE',
]:
disconnected_suppliers.add(supplier_idurl)
# elif sc.state in ['QUEUE?', 'REQUEST', ]:
# requested_suppliers.add(supplier_idurl)
if contact_status.isOffline(supplier_idurl):
offline_suppliers.add(supplier_idurl)
elif contact_status.isOnline(supplier_idurl):
online_suppliers.add(supplier_idurl)
elif contact_status.isCheckingNow(supplier_idurl):
requested_suppliers.add(supplier_idurl)
if contactsdb.num_suppliers() > number_desired:
for supplier_index in range(number_desired,
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
lg.warn('found "REDUNDANT" supplier %s at position %d' % (
idurl,
supplier_index,
))
potentialy_fired.add(idurl)
redundant_suppliers.add(idurl)
else:
lg.warn('supplier at position %d not exist' %
supplier_index)
if not connected_suppliers or not online_suppliers:
lg.warn('SKIP, no ONLINE suppliers found at the moment')
self.automat('made-decision', [])
return
if requested_suppliers:
lg.warn('SKIP, still waiting response from some of suppliers')
self.automat('made-decision', [])
return
if redundant_suppliers:
result = list(redundant_suppliers)
lg.info('will replace redundant suppliers: %s' % result)
self.automat('made-decision', result)
return
if not disconnected_suppliers:
lg.warn('SKIP, no OFFLINE suppliers found at the moment')
# TODO: add more conditions to fire "slow" suppliers
self.automat('made-decision', [])
return
if len(offline_suppliers) + len(online_suppliers) != number_desired:
lg.warn('SKIP, offline + online != total count: %s %s %s' %
(offline_suppliers, online_suppliers, number_desired))
self.automat('made-decision', [])
return
from raid import eccmap
max_offline_suppliers_count = eccmap.GetCorrectableErrors(
number_desired)
if len(offline_suppliers) > max_offline_suppliers_count:
lg.warn(
'SKIP, too many OFFLINE suppliers at the moment : %d > %d' % (
len(offline_suppliers),
max_offline_suppliers_count,
))
self.automat('made-decision', [])
return
critical_offline_suppliers_count = eccmap.GetFireHireErrors(
number_desired)
# TODO: temporary disabled because of an issue: too aggressive replacing suppliers who still have the data
if False: # len(offline_suppliers) >= critical_offline_suppliers_count:
one_dead_supplier = offline_suppliers.pop()
lg.warn(
'found "CRITICALLY_OFFLINE" supplier %s, max offline limit is %d'
% (
one_dead_supplier,
critical_offline_suppliers_count,
))
potentialy_fired.add(one_dead_supplier)
if not potentialy_fired:
lg.out(
6,
'fire_hire.doDecideToDismiss found no "bad" suppliers, all is good !!!!!'
)
self.automat('made-decision', [])
return
# only replace suppliers one by one at the moment
result = list(potentialy_fired)
lg.info('will replace supplier %s' % result[0])
self.automat('made-decision', [
result[0],
])
def doSubstituteSupplier(self, *args, **kwargs):
"""
Action method.
"""
new_idurl = id_url.field(args[0])
family_position = kwargs.get('family_position')
current_suppliers = list(contactsdb.suppliers())
desired_suppliers = settings.getSuppliersNumberDesired()
old_idurl = None
if family_position in self.hire_list:
self.hire_list.remove(family_position)
lg.info(
'found position on which new supplier suppose to be hired: %d'
% family_position)
else:
lg.warn('did not found position for new supplier to be hired on')
if new_idurl in current_suppliers:
raise Exception('%s is already supplier' % new_idurl)
if family_position is None or family_position == -1:
lg.warn(
'unknown family_position from supplier results, will pick first empty spot'
)
position = -1
old_idurl = None
for i in range(len(current_suppliers)):
if not current_suppliers[i].strip():
position = i
break
if id_url.is_in(current_suppliers[i],
self.dismiss_list,
as_field=False):
position = i
old_idurl = current_suppliers[i]
break
family_position = position
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doSubstituteSupplier family_position=%d' %
family_position)
contactsdb.add_supplier(idurl=new_idurl, position=family_position)
contactsdb.save_suppliers()
misc.writeSupplierData(
new_idurl,
'connected',
time.strftime('%d-%m-%Y %H:%M:%S'),
my_id.getIDURL(),
)
from main import control
control.on_suppliers_changed(current_suppliers)
if family_position < 0:
lg.info(
'added new supplier, family position unknown: %s desired_suppliers=%d current_suppliers=%d'
% (new_idurl, desired_suppliers, len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=None,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
else:
if old_idurl:
lg.info(
'hired new supplier and substitute existing supplier on position %d : %s->%s desired_suppliers=%d current_suppliers=%d'
% (family_position, old_idurl, new_idurl,
desired_suppliers, len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=old_idurl,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
else:
lg.info(
'hired new supplier on empty position %d : %s desired_suppliers=%d current_suppliers=%d'
% (family_position, new_idurl, desired_suppliers,
len(contactsdb.suppliers())))
events.send('supplier-modified',
data=dict(
new_idurl=new_idurl,
old_idurl=None,
position=family_position,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(
contactsdb.suppliers()),
))
self.restart_interval = 1.0
if _Debug:
lg.out(_DebugLevel,
' my current suppliers: %r' % contactsdb.suppliers())