def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(10,
'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
result = list(result)
lg.out(10, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def doOverallCheckUp(self, *args, **kwargs):
"""
Action method.
"""
if '' in contactsdb.suppliers() or b'' in contactsdb.suppliers():
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doOverallCheckUp found empty supplier, restart now'
)
self.automat('restart')
return
if online_status.listOfflineSuppliers():
if time.time() - self.last_execution_time > 60:
# re-sync every 1 min. if at least on supplier is dead
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doOverallCheckUp restart after 1 min, found offline suppliers'
)
self.automat('restart')
return
if time.time() - self.last_execution_time > 60 * 10:
# also re-sync every 10 min.
if _Debug:
lg.out(
_DebugLevel,
'backup_monitor.doOverallCheckUp periodic 10 min. restart'
)
self.automat('restart')
return
def isMoreNeeded(self, *args, **kwargs):
"""
Condition method.
"""
# if _Debug:
# lg.out(_DebugLevel, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
# contactsdb.num_suppliers(), len(self.dismiss_list),
# settings.getSuppliersNumberDesired()))
if id_url.is_some_empty(contactsdb.suppliers()):
if _Debug:
lg.out(_DebugLevel,
'fire_hire.isMoreNeeded found empty supplier!!!')
return True
if isinstance(args[0], list):
dismissed = args[0]
else:
dismissed = self.dismiss_list
s = set(id_url.to_bin_list(contactsdb.suppliers()))
s.difference_update(set(id_url.to_bin_list(dismissed)))
result = len(s) < settings.getSuppliersNumberDesired()
if _Debug:
lg.out(
_DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' %
(contactsdb.num_suppliers(), len(dismissed), len(s),
settings.getSuppliersNumberDesired(), result))
return result
def isMoreNeeded(self, arg):
"""
Condition method.
"""
# lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
# contactsdb.num_suppliers(), len(self.dismiss_list),
# settings.getSuppliersNumberDesired()))
if '' in contactsdb.suppliers():
if _Debug:
lg.out(_DebugLevel,
'fire_hire.isMoreNeeded found empty suppliers!!!')
return True
if isinstance(arg, list):
dismissed = arg
else:
dismissed = self.dismiss_list
s = set(contactsdb.suppliers())
s.difference_update(set(dismissed))
result = len(s) < settings.getSuppliersNumberDesired()
if _Debug:
lg.out(
_DebugLevel, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' %
(contactsdb.num_suppliers(), len(dismissed), len(s),
settings.getSuppliersNumberDesired(), result))
return result
def doDecideToDismiss(self, arg):
"""
Action method.
"""
global _SuppliersToFire
result = set(_SuppliersToFire)
_SuppliersToFire = []
# if you have some empty suppliers need to get rid of them,
# but no need to dismiss anyone at the moment.
if '' in contactsdb.suppliers():
lg.out(10, 'fire_hire.doDecideToDismiss found empty supplier, SKIP')
self.automat('made-decision', [])
return
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if not sc:
continue
if sc.state == 'NO_SERVICE':
result.add(supplier_idurl)
if contactsdb.num_suppliers() > settings.getSuppliersNumberDesired():
for supplier_index in range(
settings.getSuppliersNumberDesired(),
contactsdb.num_suppliers()):
idurl = contactsdb.supplier(supplier_index)
if idurl:
result.add(idurl)
result = list(result)
lg.out(10, 'fire_hire.doDecideToDismiss %s' % result)
self.automat('made-decision', result)
def RunTask():
"""
Checks current jobs and run a one task if it is possible.
Verifies if it is possible to start a new task,
the maximum number of simultaneously running ``Jobs`` is limited.
"""
if len(tasks()) == 0:
return False
if len(jobs()) >= MAXIMUM_JOBS_STARTED:
return False
if b'' in contactsdb.suppliers() or '' in contactsdb.suppliers():
if _Debug:
lg.out(
_DebugLevel,
'backup_control.RunTask found empty supplier, retry after 5 sec'
)
reactor.callLater(5, RunTask) # @UndefinedVariable
return False
T = tasks().pop(0)
message = T.run()
if message:
events.send('backup-task-failed',
data=dict(
path_id=T.pathID,
message=message,
))
T.result_defer.errback((T.pathID, message))
else:
# events.send('backup-task-executed', data=dict(path_id=T.pathID, backup_id=T.backupID, ))
T.result_defer.callback((T.backupID, None))
T.destroy(message)
return True
def doFindNewSupplier(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'fire_hire.doFindNewSupplier desired_suppliers=%d current_suppliers=%r'
%
(settings.getSuppliersNumberDesired(), contactsdb.suppliers()))
from p2p import network_connector
if network_connector.A().state != 'CONNECTED':
if _Debug:
lg.out(
_DebugLevel,
' network_connector is not CONNECTED at the moment, SKIP'
)
self.automat('search-failed')
return
position_for_new_supplier = None
for pos in range(settings.getSuppliersNumberDesired()):
if pos in self.hire_list:
continue
supplier_idurl = contactsdb.supplier(pos)
if not supplier_idurl:
lg.info(
'found empty supplier at position %d and going to find new supplier on that position'
% pos)
position_for_new_supplier = pos
break
if id_url.is_in(supplier_idurl, self.dismiss_list, as_field=False):
lg.info(
'going to find new supplier on existing position %d to replace supplier %s'
% (
pos,
supplier_idurl,
))
position_for_new_supplier = pos
break
if position_for_new_supplier is None:
lg.err('did not found position for new supplier')
self.automat('search-failed')
return
from customer import supplier_finder
for idurl_txt in strng.to_text(config.conf().getData(
'services/employer/candidates')).split(','):
if idurl_txt.strip():
supplier_finder.AddSupplierToHire(idurl_txt)
self.hire_list.append(position_for_new_supplier)
supplier_finder.A(
'start',
family_position=position_for_new_supplier,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(contactsdb.suppliers()),
)
def doRemoveSuppliers(self, *args, **kwargs):
"""
Action method.
"""
current_suppliers = contactsdb.suppliers()
desired_suppliers = settings.getSuppliersNumberDesired()
if len(current_suppliers) < desired_suppliers:
lg.warn('must have more suppliers %d<%d' %
(len(current_suppliers), desired_suppliers))
removed_suppliers = []
for supplier_idurl in self.dismiss_list:
if id_url.is_not_in(supplier_idurl,
current_suppliers,
as_field=False):
lg.warn('%s not a supplier' % supplier_idurl)
continue
pos = current_suppliers.index(id_url.field(supplier_idurl))
current_suppliers[pos] = ''
removed_suppliers.append((
pos,
supplier_idurl,
))
misc.writeSupplierData(
supplier_idurl,
'disconnected',
time.strftime('%d-%m-%Y %H:%M:%S'),
my_id.getLocalID(),
)
current_suppliers = current_suppliers[:desired_suppliers]
contactsdb.update_suppliers(current_suppliers)
contactsdb.save_suppliers()
from main import control
control.on_suppliers_changed(current_suppliers)
for position, supplier_idurl in removed_suppliers:
events.send(
'supplier-modified',
dict(
new_idurl=None,
old_idurl=supplier_idurl,
position=position,
))
lg.info(
'removed some suppliers : %d desired_suppliers=%d current_suppliers=%d'
% (len(self.dismiss_list), desired_suppliers,
len(contactsdb.suppliers())))
if _Debug:
lg.out(_DebugLevel,
' my current suppliers: %r' % contactsdb.suppliers())
def doInit(self, arg):
"""
Action method.
"""
if data_receiver.A():
data_receiver.A().addStateChangedCallback(
self._on_data_receiver_state_changed)
self.known_suppliers = filter(
None, contactsdb.suppliers(customer_idurl=self.customer_idurl))
known_eccmap_dict = {}
for supplier_idurl in self.known_suppliers:
known_ecc_map = contactsdb.get_supplier_meta_info(
supplier_idurl=supplier_idurl,
customer_idurl=self.customer_idurl,
).get('ecc_map', None)
if known_ecc_map:
if known_ecc_map not in known_eccmap_dict:
known_eccmap_dict[known_ecc_map] = 0
known_eccmap_dict[known_ecc_map] += 1
if known_eccmap_dict:
all_known_eccmaps = known_eccmap_dict.items()
all_known_eccmaps.sort(key=lambda i: i[1], reverse=True)
self.EccMap = eccmap.eccmap(all_known_eccmaps[0][0])
lg.info('eccmap %s recognized from suppliers meta info' %
self.EccMap)
else:
self.EccMap = eccmap.eccmap(
eccmap.GetEccMapName(len(self.known_suppliers)))
lg.warn(
'no meta info found, guessed eccmap %s from %d known suppliers'
% (self.EccMap, len(self.known_suppliers)))
def health_check(self):
from raid import eccmap
from contacts import contactsdb
from userid import id_url
missed_suppliers = id_url.empty_count(contactsdb.suppliers())
# to have that service running minimum amount of suppliers must be already in the family
return missed_suppliers <= eccmap.Current().CorrectableErrors
def _do_request(self, x=None):
from raid import eccmap
self.received_lf_counter = 0
self.requested_lf_packet_ids.clear()
known_suppliers = contactsdb.suppliers(customer_idurl=self.target_customer_idurl)
try:
self.critical_suppliers_number = eccmap.GetCorrectableErrors(len(known_suppliers))
except:
lg.warn('number of known suppliers for customer %r is not standard' % self.target_customer_idurl)
self.critical_suppliers_number = int(float(len(known_suppliers)) * 0.75)
for idurl in known_suppliers:
if idurl:
if online_status.isOnline(idurl):
if _Debug:
lg.out(_DebugLevel, 'list_files_orator._do_request ListFiles() from my supplier %s' % idurl)
outpacket = p2p_service.SendListFiles(
target_supplier=idurl,
customer_idurl=self.target_customer_idurl,
timeout=30,
)
if outpacket:
self.requested_lf_packet_ids.add(outpacket.PacketID)
else:
lg.err('failed sending ListFiles() to %r' % idurl)
else:
lg.warn('skip sending ListFiles() because %s is not online' % idurl)
def doInit(self, *args, **kwargs):
"""
Action method.
"""
self._do_block_rebuilding()
self.known_suppliers = [
_f
for _f in contactsdb.suppliers(customer_idurl=self.customer_idurl)
if _f
]
if not self.EccMap:
if self.customer_idurl == my_id.getIDURL():
self.EccMap = eccmap.Current()
lg.info('ECC map %r set from local for my own suppliers' %
self.EccMap)
if not self.EccMap:
known_eccmap_dict = {}
for supplier_idurl in self.known_suppliers:
known_ecc_map = contactsdb.get_supplier_meta_info(
supplier_idurl=supplier_idurl,
customer_idurl=self.customer_idurl,
).get('ecc_map', None)
if known_ecc_map:
if known_ecc_map not in known_eccmap_dict:
known_eccmap_dict[known_ecc_map] = 0
known_eccmap_dict[known_ecc_map] += 1
if known_eccmap_dict:
all_known_eccmaps = list(known_eccmap_dict.items())
all_known_eccmaps.sort(key=lambda i: i[1], reverse=True)
self.EccMap = eccmap.eccmap(all_known_eccmaps[0][0])
lg.info('ECC map %r recognized from suppliers meta info' %
self.EccMap)
else:
known_ecc_map = None
if driver.is_on('service_shared_data'):
from access import shared_access_coordinator
active_share = shared_access_coordinator.get_active_share(
self.key_id)
if active_share:
known_ecc_map = active_share.known_ecc_map
if known_ecc_map:
self.EccMap = eccmap.eccmap(known_ecc_map)
lg.info('ECC map %r recognized from active share %r' % (
self.EccMap,
active_share,
))
else:
num_suppliers = len(self.known_suppliers)
if num_suppliers not in eccmap.GetPossibleSuppliersCount():
num_suppliers = settings.DefaultDesiredSuppliers()
self.EccMap = eccmap.eccmap(
eccmap.GetEccMapName(num_suppliers))
lg.warn(
'no meta info found, guessed ECC map %r from %d known suppliers'
% (self.EccMap, len(self.known_suppliers)))
self.max_errors = eccmap.GetCorrectableErrors(
self.EccMap.NumSuppliers())
if data_receiver.A():
data_receiver.A().addStateChangedCallback(
self._on_data_receiver_state_changed)
def doSuppliersSendDBInfo(self, arg):
from p2p import contact_status
lg.out(4, 'backup_db_keeper.doSuppliersSendDBInfo')
packetID = settings.BackupIndexFileName()
self.sentSuppliers.clear()
src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = my_id.getLocalID()
b = encrypted.Block(
localID,
packetID,
0,
key.NewSessionKey(),
key.SessionKeyType(),
True,
src)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = signed.Packet(
commands.Data(), localID, localID, packetID,
Payload, supplierId)
pkt_out = gateway.outbox(newpacket, callbacks={
commands.Ack(): self._supplier_acked,
commands.Fail(): self._supplier_acked, })
self.sentSuppliers.add(supplierId)
lg.out(
4, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def _do_retrieve(self, x=None):
packetID = global_id.MakeGlobalID(
customer=my_id.getGlobalID(key_alias='master'),
path=settings.BackupIndexFileName(),
)
localID = my_id.getIDURL()
for supplier_idurl in contactsdb.suppliers():
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None or sc.state != 'CONNECTED':
continue
if online_status.isOffline(supplier_idurl):
continue
pkt_out = p2p_service.SendRetreive(ownerID=localID,
creatorID=localID,
packetID=packetID,
remoteID=supplier_idurl,
response_timeout=60 * 2,
callbacks={
commands.Data():
self._on_supplier_response,
commands.Fail():
self._on_supplier_fail,
})
if pkt_out:
self.requesting_suppliers.add(supplier_idurl)
self.requested_suppliers_number += 1
self.requests_packets_sent.append((packetID, supplier_idurl))
if _Debug:
lg.out(
_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplier_idurl)))
def RequestListFilesAll(customer_idurl=None):
r = []
for supplier_idurl in contactsdb.suppliers(customer_idurl=customer_idurl):
r.append(
SendRequestListFiles(supplier_idurl,
customer_idurl=customer_idurl))
return r
def doSupplierConnect(self, *args, **kwargs):
"""
Action method.
"""
from customer import supplier_connector
from customer import fire_hire
from raid import eccmap
position = self.family_position
if not position:
lg.warn('position for new supplier is unknown, will "guess"')
current_suppliers = list(contactsdb.suppliers())
for i in range(len(current_suppliers)):
if not current_suppliers[i].strip():
position = i
break
if current_suppliers[i] in fire_hire.A().dismiss_list:
position = i
break
sc = supplier_connector.by_idurl(self.target_idurl)
if not sc:
sc = supplier_connector.create(
supplier_idurl=self.target_idurl,
customer_idurl=my_id.getLocalID(),
)
sc.automat(
'connect',
family_position=position,
ecc_map=(self.ecc_map or eccmap.Current().name),
family_snapshot=self.family_snapshot,
)
sc.set_callback('supplier_finder', self._supplier_connector_state)
def doConnectSuppliers(self, *args, **kwargs):
"""
Action method.
"""
self.connect_list = []
my_current_family = list(contactsdb.suppliers())
for pos, supplier_idurl in enumerate(my_current_family):
if not supplier_idurl:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None:
sc = supplier_connector.create(
supplier_idurl=supplier_idurl,
customer_idurl=my_id.getLocalID(),
)
sc.set_callback('fire_hire',
self._on_supplier_connector_state_changed)
self.connect_list.append(supplier_idurl)
sc.automat(
'connect',
family_position=pos,
ecc_map=eccmap.Current().name,
family_snapshot=my_current_family,
)
supplier_contact_status = contact_status.getInstance(
supplier_idurl)
if supplier_contact_status:
supplier_contact_status.addStateChangedCallback(
self._on_supplier_contact_status_state_changed,
newstate='OFFLINE',
)
def _stats(params):
from contacts import contactsdb
from p2p import contact_status
from lib import diskspace
result = {}
result['suppliers'] = contactsdb.num_suppliers()
result['max_suppliers'] = settings.getSuppliersNumberDesired()
result['online_suppliers'] = contact_status.countOnlineAmong(
contactsdb.suppliers())
result['customers'] = contactsdb.num_customers()
result['bytes_donated'] = settings.getDonatedBytes()
result['value_donated'] = diskspace.MakeStringFromBytes(
settings.getDonatedBytes())
result['bytes_needed'] = settings.getNeededBytes()
result['value_needed'] = diskspace.MakeStringFromBytes(
settings.getNeededBytes())
result['bytes_used_total'] = backup_fs.sizebackups()
result['value_used_total'] = diskspace.MakeStringFromBytes(
backup_fs.sizebackups())
result['bytes_used_supplier'] = 0 if (
contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() /
contactsdb.num_suppliers()))
result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
result['files_count'] = backup_fs.numberfiles()
result['folders_count'] = backup_fs.numberfolders()
result['items_count'] = backup_fs.counter()
result['timestamp'] = time.time()
return {
'result': result,
}
def ping_suppliers(customer_idurl=None, timeout=20):
from p2p import online_status
l = []
for supplier_idurl in contactsdb.suppliers(customer_idurl=customer_idurl):
if supplier_idurl:
l.append(online_status.ping(idurl=supplier_idurl, ack_timeout=timeout, channel='ping_suppliers', keep_alive=True))
return DeferredList(l, consumeErrors=True)
def SendSuppliers(customer_idurl=None):
"""
Send my identity file to all my suppliers, calls to ``SendToIDs()`` method.
"""
lg.out(6, "propagate.SendSuppliers")
SendToIDs(contactsdb.suppliers(customer_idurl=customer_idurl),
HandleSuppliersAck)
def doRemoveSuppliers(self, *args, **kwargs):
"""
Action method.
"""
current_suppliers = contactsdb.suppliers()
desired_suppliers = settings.getSuppliersNumberDesired()
if len(current_suppliers) < desired_suppliers:
lg.warn('must have more suppliers %d<%d' % (
len(current_suppliers), desired_suppliers))
removed_suppliers = []
for supplier_idurl in self.dismiss_list:
if supplier_idurl not in current_suppliers:
lg.warn('%s not a supplier' % supplier_idurl)
continue
pos = current_suppliers.index(supplier_idurl)
# current_suppliers.remove(supplier_idurl)
current_suppliers[pos] = ''
removed_suppliers.append((pos, supplier_idurl,))
misc.writeSupplierData(
supplier_idurl,
'disconnected',
time.strftime('%d-%m-%Y %H:%M:%S'),
my_id.getLocalID(),
)
current_suppliers = current_suppliers[:desired_suppliers]
contactsdb.update_suppliers(current_suppliers)
contactsdb.save_suppliers()
from main import control
control.on_suppliers_changed(current_suppliers)
for position, supplier_idurl in removed_suppliers:
events.send('supplier-modified', dict(
new_idurl=None, old_idurl=supplier_idurl, position=position,
))
lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def SendSuppliers(customer_idurl=None):
"""
Send my identity file to all my suppliers, calls to ``SendToIDs()`` method.
"""
if _Debug:
lg.out(_DebugLevel, "propagate.SendSuppliers")
SendToIDs(contactsdb.suppliers(customer_idurl=customer_idurl), ack_handler=HandleSuppliersAck, wide=True)
def doRemoveSuppliers(self, arg):
"""
Action method.
"""
current_suppliers = contactsdb.suppliers()
desired_suppliers = settings.getSuppliersNumberDesired()
if len(current_suppliers) < desired_suppliers:
lg.warn('must have more suppliers %d<%d' % (
len(current_suppliers), desired_suppliers))
for supplier_idurl in self.dismiss_list:
if supplier_idurl not in current_suppliers:
lg.warn('%s not a supplier' % supplier_idurl)
continue
pos = current_suppliers.index(supplier_idurl)
# current_suppliers.remove(supplier_idurl)
current_suppliers[pos] = ''
misc.writeSupplierData(
supplier_idurl,
'disconnected',
time.strftime('%d-%m-%Y %H:%M:%S'))
current_suppliers = current_suppliers[:desired_suppliers]
contactsdb.update_suppliers(current_suppliers)
contactsdb.save_suppliers()
if settings.NewWebGUI():
from web import control
control.on_suppliers_changed(current_suppliers)
else:
from web import webcontrol
webcontrol.OnListSuppliers()
lg.out(2, '!!!!!!!!!!! REMOVE SUPPLIERS : %d' % len(self.dismiss_list))
def doFindNewSupplier(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'fire_hire.doFindNewSupplier')
position_for_new_supplier = None
for pos in range(settings.getSuppliersNumberDesired()):
if pos in self.hire_list:
continue
supplier_idurl = contactsdb.supplier(pos)
if not supplier_idurl:
lg.info('found empty supplier at position %d and going to find new supplier on that position' % pos)
position_for_new_supplier = pos
break
if supplier_idurl in self.dismiss_list:
lg.info('going to find new supplier on existing position %d to replace supplier %s' % (
pos, supplier_idurl, ))
position_for_new_supplier = pos
break
if position_for_new_supplier is None:
lg.err('did not found position for new supplier')
self.automat('search-failed')
return
self.hire_list.append(position_for_new_supplier)
supplier_finder.A(
'start',
family_position=position_for_new_supplier,
ecc_map=eccmap.Current().name,
family_snapshot=contactsdb.suppliers(),
)
def doRememberSuppliers(self, *args, **kwargs):
"""
Action method.
"""
self.current_suppliers = list(contactsdb.suppliers())
self.backups_progress_last_iteration = 0
self.last_execution_time = time.time()
def doSupplierConnect(self, *args, **kwargs):
"""
Action method.
"""
from customer import supplier_connector
from customer import fire_hire
from raid import eccmap
position = self.family_position
if position is None or position == -1:
lg.warn('position for new supplier is unknown, will "guess"')
current_suppliers = list(contactsdb.suppliers())
for i in range(len(current_suppliers)):
supplier_idurl = current_suppliers[i].to_bin()
if not supplier_idurl:
position = i
break
if id_url.is_in(supplier_idurl,
fire_hire.A().dismiss_list,
as_field=False):
position = i
break
sc = supplier_connector.by_idurl(self.target_idurl)
if not sc:
sc = supplier_connector.create(
supplier_idurl=self.target_idurl,
customer_idurl=my_id.getIDURL(),
)
sc.set_callback('supplier_finder', self._supplier_connector_state)
sc.automat(
'connect',
family_position=position,
ecc_map=(self.ecc_map or eccmap.Current().name),
family_snapshot=self.family_snapshot,
)
def isStillNeeded(self, *args, **kwargs):
"""
Condition method.
"""
supplier_idurl = args[0]
current_suppliers = contactsdb.suppliers()
if supplier_idurl in current_suppliers:
# this guy is already a supplier, we still need more then
return True
desired_number = settings.getSuppliersNumberDesired()
needed_suppliers = current_suppliers[:desired_number]
empty_suppliers = needed_suppliers.count(id_url.field(b''))
# if '' in needed_suppliers:
# lg.warn('found empty suppliers!!!')
# return True
s = set(id_url.to_bin_list(needed_suppliers))
s.add(id_url.to_bin(supplier_idurl))
s.difference_update(set(id_url.to_bin_list(self.dismiss_list)))
result = len(s) - empty_suppliers < settings.getSuppliersNumberDesired(
)
# if _Debug:
# lg.out(_DebugLevel, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % (
# contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list),
# len(s), settings.getSuppliersNumberDesired(), result))
return result
def doUpdateSuppliers(self, *args, **kwargs):
"""
Action method.
"""
from customer import io_throttle
# supplierList = contactsdb.suppliers()
# take a list of suppliers positions that was changed
changedSupplierNums = backup_matrix.SuppliersChangedNumbers(
self.current_suppliers)
# notify io_throttle that we do not neeed already this suppliers
for supplierNum in changedSupplierNums:
lg.out(
2,
"backup_monitor.doUpdateSuppliers supplier %d changed: [%s]->[%s]"
% (
supplierNum,
nameurl.GetName(self.current_suppliers[supplierNum]),
nameurl.GetName(contactsdb.suppliers()[supplierNum]),
))
suplier_idurl = self.current_suppliers[supplierNum]
io_throttle.DeleteSuppliers([
suplier_idurl,
])
# erase (set to 0) remote info for this guys
backup_matrix.ClearSupplierRemoteInfo(supplierNum)
def isExistSomeSuppliers(self, *args, **kwargs):
"""
Condition method.
"""
sup_list = contactsdb.suppliers()
return contactsdb.num_suppliers() > 0 and sup_list.count(
id_url.field(b'')) < contactsdb.num_suppliers()
def doConnectSuppliers(self, *args, **kwargs):
"""
Action method.
"""
from customer import supplier_connector
from p2p import online_status
self.connect_list = []
my_current_family = contactsdb.suppliers()
for pos, supplier_idurl in enumerate(my_current_family):
if not supplier_idurl:
continue
if self.configs[0] and pos >= self.configs[0]:
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None:
sc = supplier_connector.create(
supplier_idurl=supplier_idurl,
customer_idurl=my_id.getIDURL(),
)
else:
sc.needed_bytes = None
sc.do_calculate_needed_bytes()
sc.set_callback('fire_hire',
self._on_supplier_connector_state_changed)
self.connect_list.append(supplier_idurl)
sc.automat(
'connect',
family_position=pos,
ecc_map=eccmap.Current().name,
family_snapshot=id_url.to_bin_list(my_current_family),
)
online_status.add_online_status_listener_callback(
idurl=supplier_idurl,
callback_method=self._on_supplier_online_status_state_changed,
)
def suppliers(AckHandler=None, wide=False, customer_idurl=None):
"""
Call ``propagate()`` for all suppliers.
"""
lg.out(6, 'propagate.suppliers')
return propagate(contactsdb.suppliers(customer_idurl=customer_idurl),
AckHandler, wide)
def isAnyDataComming(self, arg):
"""
Condition method.
"""
from transport import packet_in
related_packets = packet_in.search(sender_idurl=contactsdb.suppliers())
return len(related_packets) > 0
def start(self):
from customer import supplier_connector
from contacts import contactsdb
for supplier_idurl in contactsdb.suppliers():
if supplier_idurl and not supplier_connector.by_idurl(
supplier_idurl):
supplier_connector.create(supplier_idurl)
return True
def doOverallCheckUp(self, arg):
"""
Action method.
"""
if '' in contactsdb.suppliers():
lg.out(6, 'backup_monitor.doOverallCheckUp found empty supplier, restart now')
self.automat('restart')
return
def doDHTFindRandomUser(self, *args, **kwargs):
"""
Action method.
"""
tsk = lookup.random_supplier(ignore_idurls=contactsdb.suppliers())
tsk.result_defer.addCallback(self._nodes_lookup_finished)
tsk.result_defer.addErrback(
lambda err: self.automat('users-not-found'))
def isMoreNeeded(self, arg):
"""
Condition method.
"""
# lg.out(10, 'fire_hire.isMoreNeeded current=%d dismiss=%d needed=%d' % (
# contactsdb.num_suppliers(), len(self.dismiss_list),
# settings.getSuppliersNumberDesired()))
if '' in contactsdb.suppliers():
lg.out(4, 'fire_hire.isMoreNeeded found empty suppliers!!!')
return True
if isinstance(arg, list):
dismissed = arg
else:
dismissed = self.dismiss_list
s = set(contactsdb.suppliers())
s.difference_update(set(dismissed))
result = len(s) < settings.getSuppliersNumberDesired()
lg.out(14, 'fire_hire.isMoreNeeded %d %d %d %d, result=%s' % (
contactsdb.num_suppliers(), len(dismissed), len(s),
settings.getSuppliersNumberDesired(), result))
return result
def listOfflineSuppliers():
"""
Loops all suppliers and check their state, return a list of those with
state OFFLINE.
"""
result = []
for idurl in contactsdb.suppliers():
if not idurl:
result.append(idurl)
elif isOffline(idurl):
result.append(idurl)
return result
def doRequestRemoteFiles(self, arg):
global _RequestedListFilesCounter
global _RequestedListFilesPacketIDs
_RequestedListFilesCounter = 0
_RequestedListFilesPacketIDs.clear()
for idurl in contactsdb.suppliers():
if idurl:
if contact_status.isOnline(idurl):
p2p_service.SendRequestListFiles(idurl)
_RequestedListFilesPacketIDs.add(idurl)
else:
lg.out(6, 'list_files_orator.doRequestRemoteFiles SKIP %s is not online' % idurl)
def ReadLatestRawListFiles():
"""
Call ``ReadRawListFiles()`` for every local file we have on hands and build
whole "remote" matrix.
"""
lg.out(4, "backup_matrix.ReadLatestRawListFiles")
for idurl in contactsdb.suppliers():
if idurl:
filename = os.path.join(settings.SupplierPath(idurl, "listfiles"))
if os.path.isfile(filename):
listFileText = bpio.ReadTextFile(filename)
if listFileText.strip() != "":
ReadRawListFiles(contactsdb.supplier_position(idurl), listFileText)
def RequestDeleteListPaths(pathIDs):
if _Debug:
lg.out(_DebugLevel, "p2p_service.RequestDeleteListPaths wish to delete %d paths" % len(pathIDs))
for supplier in contactsdb.suppliers():
if not supplier:
continue
# found = False
# for workitem in transport_control.SendQueue():
# if workitem.command == commands.DeleteFile() and workitem.remoteid == supplier:
# found = True
# break
# if found:
# continue
SendDeleteListPaths(supplier, pathIDs)
def doSubstituteSupplier(self, arg):
"""
Action method.
"""
new_idurl = arg
current_suppliers = list(contactsdb.suppliers())
if new_idurl in current_suppliers:
raise Exception('%s is already supplier' % new_idurl)
position = -1
old_idurl = None
for i in range(len(current_suppliers)):
if current_suppliers[i].strip() == '':
position = i
break
if current_suppliers[i] in self.dismiss_list:
# self.dismiss_list.remove(current_suppliers[i])
position = i
old_idurl = current_suppliers[i]
break
lg.out(10, 'fire_hire.doSubstituteSupplier position=%d' % position)
if position < 0:
current_suppliers.append(new_idurl)
else:
current_suppliers[position] = new_idurl
contactsdb.update_suppliers(current_suppliers)
contactsdb.save_suppliers()
misc.writeSupplierData(
new_idurl,
'connected',
time.strftime('%d-%m-%Y %H:%M:%S'))
if settings.NewWebGUI():
from web import control
# control.on_suppliers_changed(current_suppliers)
else:
from web import webcontrol
webcontrol.OnListSuppliers()
if position < 0:
lg.out(2, '!!!!!!!!!!! ADD SUPPLIER : %s' % (new_idurl))
else:
if old_idurl:
lg.out(
2, '!!!!!!!!!!! SUBSTITUTE SUPPLIER %d : %s->%s' %
(position, old_idurl, new_idurl))
else:
lg.out(
2, '!!!!!!!!!!! REPLACE EMPTY SUPPLIER %d : %s' %
(position, new_idurl))
self.restart_interval = 1.0
def doConnectSuppliers(self, arg):
"""
Action method.
"""
self.connect_list = []
for supplier_idurl in contactsdb.suppliers():
if supplier_idurl == '':
continue
sc = supplier_connector.by_idurl(supplier_idurl)
if sc is None:
sc = supplier_connector.create(supplier_idurl)
sc.set_callback(
'fire_hire',
self._supplier_connector_state_changed)
self.connect_list.append(supplier_idurl)
sc.automat('connect')
def RequestDeleteBackup(BackupID):
"""
Need to send a "DeleteBackup" command to all suppliers.
"""
if _Debug:
lg.out(_DebugLevel, "p2p_service.RequestDeleteBackup with BackupID=" + str(BackupID))
for supplier in contactsdb.suppliers():
if not supplier:
continue
# prevItems = [] # transport_control.SendQueueSearch(BackupID)
# found = False
# for workitem in prevItems:
# if workitem.remoteid == supplier:
# found = True
# break
# if found:
# continue
SendDeleteBackup(supplier, BackupID)
def doUpdateSuppliers(self, arg):
"""
Action method.
"""
from customer import io_throttle
# supplierList = contactsdb.suppliers()
# take a list of suppliers positions that was changed
changedSupplierNums = backup_matrix.SuppliersChangedNumbers(self.current_suppliers)
# notify io_throttle that we do not neeed already this suppliers
for supplierNum in changedSupplierNums:
lg.out(2, "backup_monitor.doUpdateSuppliers supplier %d changed: [%s]->[%s]" % (
supplierNum,
nameurl.GetName(self.current_suppliers[supplierNum]),
nameurl.GetName(contactsdb.suppliers()[supplierNum]),))
suplier_idurl = self.current_suppliers[supplierNum]
io_throttle.DeleteSuppliers([suplier_idurl, ])
# erase (set to 0) remote info for this guys
backup_matrix.ClearSupplierRemoteInfo(supplierNum)
def doSuppliersRequestDBInfo(self, arg):
lg.out(4, 'backup_db_keeper.doSuppliersRequestDBInfo')
packetID = settings.BackupIndexFileName()
self.requestedSuppliers.clear()
Payload = ''
localID = my_id.getLocalID()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
newpacket = signed.Packet(
commands.Retrieve(),
localID,
localID,
packetID,
Payload,
supplierId)
gateway.outbox(newpacket, callbacks={
commands.Data(): self._supplier_response,
commands.Fail(): self._supplier_response, })
self.requestedSuppliers.add(supplierId)
def _stats(params):
from contacts import contactsdb
from p2p import contact_status
from lib import diskspace
result = {}
result['suppliers'] = contactsdb.num_suppliers()
result['max_suppliers'] = settings.getSuppliersNumberDesired()
result['online_suppliers'] = contact_status.countOnlineAmong(contactsdb.suppliers())
result['customers'] = contactsdb.num_customers()
result['bytes_donated'] = settings.getDonatedBytes()
result['value_donated'] = diskspace.MakeStringFromBytes(settings.getDonatedBytes())
result['bytes_needed'] = settings.getNeededBytes()
result['value_needed'] = diskspace.MakeStringFromBytes(settings.getNeededBytes())
result['bytes_used_total'] = backup_fs.sizebackups()
result['value_used_total'] = diskspace.MakeStringFromBytes(backup_fs.sizebackups())
result['bytes_used_supplier'] = 0 if (contactsdb.num_suppliers() == 0) else (int(backup_fs.sizebackups() / contactsdb.num_suppliers()))
result['bytes_indexed'] = backup_fs.sizefiles() + backup_fs.sizefolders()
result['files_count'] = backup_fs.numberfiles()
result['folders_count'] = backup_fs.numberfolders()
result['items_count'] = backup_fs.counter()
result['timestamp'] = time.time()
return {'result': result, }
def doSuppliersRequestIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersRequestIndexFile')
if driver.is_started('service_backups'):
from storage import backup_control
self.current_local_revision = backup_control.revision()
else:
self.current_local_revision = -1
self.latest_supplier_revision = -1
self.requesting_suppliers.clear()
self.requested_suppliers_number = 0
packetID = settings.BackupIndexFileName()
localID = my_id.getLocalID()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = signed.Packet(
commands.Retrieve(),
localID,
localID,
packetID,
'',
supplierId)
pkt_out = gateway.outbox(newpacket, callbacks={
commands.Data(): self._on_supplier_response,
commands.Fail(): self._on_supplier_response, })
if pkt_out:
self.requesting_suppliers.add(supplierId)
self.requested_suppliers_number += 1
if _Debug:
lg.out(_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def doSuppliersSendIndexFile(self, arg):
"""
Action method.
"""
if _Debug:
lg.out(_DebugLevel, 'index_synchronizer.doSuppliersSendIndexFile')
packetID = settings.BackupIndexFileName()
self.sending_suppliers.clear()
self.sent_suppliers_number = 0
src = bpio.ReadBinaryFile(settings.BackupIndexFilePath())
localID = my_id.getLocalID()
b = encrypted.Block(
localID,
packetID,
0,
key.NewSessionKey(),
key.SessionKeyType(),
True,
src)
Payload = b.Serialize()
for supplierId in contactsdb.suppliers():
if not supplierId:
continue
if not contact_status.isOnline(supplierId):
continue
newpacket = signed.Packet(
commands.Data(), localID, localID, packetID,
Payload, supplierId)
pkt_out = gateway.outbox(newpacket, callbacks={
commands.Ack(): self._on_supplier_acked,
commands.Fail(): self._on_supplier_acked, })
if pkt_out:
self.sending_suppliers.add(supplierId)
self.sent_suppliers_number += 1
if _Debug:
lg.out(_DebugLevel, ' %s sending to %s' %
(pkt_out, nameurl.GetName(supplierId)))
def ScanBlocksToSend(backupID):
"""
Opposite method - search for pieces which is not yet delivered to remote suppliers.
"""
if "" in contactsdb.suppliers():
return {}
localMaxBlockNum = local_max_block_numbers().get(backupID, -1)
supplierActiveArray = GetActiveArray()
bySupplier = {}
for supplierNum in xrange(len(supplierActiveArray)):
bySupplier[supplierNum] = set()
if backupID not in remote_files():
for blockNum in xrange(localMaxBlockNum + 1):
localData = GetLocalDataArray(backupID, blockNum)
localParity = GetLocalParityArray(backupID, blockNum)
for supplierNum in xrange(len(supplierActiveArray)):
if supplierActiveArray[supplierNum] != 1:
continue
if localData[supplierNum] == 1:
bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, "Data"))
if localParity[supplierNum] == 1:
bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, "Parity"))
else:
for blockNum in xrange(localMaxBlockNum + 1):
remoteData = GetRemoteDataArray(backupID, blockNum)
remoteParity = GetRemoteParityArray(backupID, blockNum)
localData = GetLocalDataArray(backupID, blockNum)
localParity = GetLocalParityArray(backupID, blockNum)
for supplierNum in xrange(len(supplierActiveArray)):
if supplierActiveArray[supplierNum] != 1:
continue
if remoteData[supplierNum] != 1 and localData[supplierNum] == 1:
bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, "Data"))
if remoteParity[supplierNum] != 1 and localParity[supplierNum] == 1:
bySupplier[supplierNum].add(packetid.MakePacketID(backupID, blockNum, supplierNum, "Parity"))
return bySupplier
def isStillNeeded(self, arg):
"""
Condition method.
"""
supplier_idurl = arg
current_suppliers = contactsdb.suppliers()
if supplier_idurl in current_suppliers:
# this guy is already a supplier, we still need more then
return True
desired_number = settings.getSuppliersNumberDesired()
needed_suppliers = current_suppliers[:desired_number]
empty_suppliers = needed_suppliers.count('')
# if '' in needed_suppliers:
# lg.warn('found empty suppliers!!!')
# return True
s = set(needed_suppliers)
s.add(supplier_idurl)
s.difference_update(set(self.dismiss_list))
result = len(s) - \
empty_suppliers < settings.getSuppliersNumberDesired()
# lg.out(14, 'fire_hire.isStillNeeded %d %d %d %d %d, result=%s' % (
# contactsdb.num_suppliers(), len(needed_suppliers), len(self.dismiss_list),
# len(s), settings.getSuppliersNumberDesired(), result))
return result
def isExistSomeSuppliers(self, arg):
"""
Condition method.
"""
return contactsdb.num_suppliers() > 0 and contactsdb.suppliers().count(
'') < contactsdb.num_suppliers()
def doRememberSuppliers(self, arg):
"""
Action method.
"""
self.current_suppliers = list(contactsdb.suppliers())
