def start(self):
from customer import io_throttle
from customer import data_sender
from customer import data_receiver
io_throttle.init()
data_sender.A('init')
data_receiver.A('init')
return True
def stop(self):
from customer import io_throttle
from customer import data_sender
from customer import data_receiver
data_receiver.A('shutdown')
data_sender.SetShutdownFlag()
data_sender.A('shutdown')
io_throttle.shutdown()
return True
def _block_finished(self, result, params):
if not result:
lg.out(
10, 'backup_rebuilder._block_finished FAILED, blockIndex=%d' %
self.blockIndex)
reactor.callLater(0, self._finish_rebuilding)
return
try:
newData, localData, localParity, reconstructedData, reconstructedParity = result
_backupID = params[0]
_blockNumber = params[1]
except:
lg.exc()
reactor.callLater(0, self._finish_rebuilding)
return
if newData:
from storage import backup_matrix
from customer import data_sender
count = 0
customer_idurl = packetid.CustomerIDURL(_backupID)
for supplierNum in xrange(
contactsdb.num_suppliers(customer_idurl=customer_idurl)):
if localData[supplierNum] == 1 and reconstructedData[
supplierNum] == 1:
backup_matrix.LocalFileReport(None, _backupID,
_blockNumber, supplierNum,
'Data')
count += 1
if localParity[supplierNum] == 1 and reconstructedParity[
supplierNum] == 1:
backup_matrix.LocalFileReport(None, _backupID,
_blockNumber, supplierNum,
'Parity')
count += 1
self.blocksSucceed.append(_blockNumber)
data_sender.A('new-data')
lg.out(
10,
'backup_rebuilder._block_finished !!!!!! %d NEW DATA segments reconstructed, blockIndex=%d'
% (count, self.blockIndex))
else:
lg.out(
10,
'backup_rebuilder._block_finished NO CHANGES, blockIndex=%d' %
self.blockIndex)
self.blockIndex -= 1
reactor.callLater(0, self._start_one_block)
def _block_finished(self, result, params):
if not result:
lg.out(10, 'backup_rebuilder._block_finished FAILED, blockIndex=%d' % self.blockIndex)
reactor.callLater(0, self._finish_rebuilding) # @UndefinedVariable
return
try:
newData, localData, localParity, reconstructedData, reconstructedParity = result
_backupID = params[0]
_blockNumber = params[1]
except:
lg.exc()
reactor.callLater(0, self._finish_rebuilding) # @UndefinedVariable
return
lg.out(10, 'backup_rebuilder._block_finished backupID=%r blockNumber=%r newData=%r' % (
_backupID, _blockNumber, newData))
lg.out(10, ' localData=%r localParity=%r' % (localData, localParity))
if newData:
from storage import backup_matrix
from customer import data_sender
count = 0
customer_idurl = packetid.CustomerIDURL(_backupID)
for supplierNum in range(contactsdb.num_suppliers(customer_idurl=customer_idurl)):
try:
localData[supplierNum]
localParity[supplierNum]
reconstructedData[supplierNum]
reconstructedParity[supplierNum]
except:
lg.err('invalid result from the task: %s' % repr(params))
lg.out(10, 'result is %s' % repr(result))
lg.exc()
continue
if localData[supplierNum] == 1 and reconstructedData[supplierNum] == 1:
backup_matrix.LocalFileReport(None, _backupID, _blockNumber, supplierNum, 'Data')
count += 1
if localParity[supplierNum] == 1 and reconstructedParity[supplierNum] == 1:
backup_matrix.LocalFileReport(None, _backupID, _blockNumber, supplierNum, 'Parity')
count += 1
self.blocksSucceed.append(_blockNumber)
data_sender.A('new-data')
lg.out(10, ' !!!!!! %d NEW DATA segments reconstructed, blockIndex=%d' % (
count, self.blockIndex))
else:
lg.out(10, ' NO CHANGES, blockIndex=%d' % self.blockIndex)
self.blockIndex -= 1
reactor.callLater(0, self._start_one_block) # @UndefinedVariable
def _request_files(self):
from storage import backup_matrix
from customer import io_throttle
from customer import data_sender
self.missingPackets = 0
# here we want to request some packets before we start working to
# rebuild the missed blocks
availableSuppliers = backup_matrix.GetActiveArray(customer_idurl=self.currentCustomerIDURL)
# remember how many requests we did on this iteration
total_requests_count = 0
# at the moment I do download everything I have available and needed
if '' in contactsdb.suppliers(customer_idurl=self.currentCustomerIDURL):
lg.out(8, 'backup_rebuilder._request_files SKIP - empty supplier')
self.automat('no-requests')
return
for supplierNum in range(contactsdb.num_suppliers(customer_idurl=self.currentCustomerIDURL)):
supplierID = contactsdb.supplier(supplierNum, customer_idurl=self.currentCustomerIDURL)
if not supplierID:
continue
requests_count = 0
# we do requests in reverse order because we start rebuilding from
# the last block
for blockIndex in range(len(self.workingBlocksQueue) - 1, -1, -1):
blockNum = self.workingBlocksQueue[blockIndex]
# do not keep too many requests in the queue
if io_throttle.GetRequestQueueLength(supplierID) >= 16:
break
# also don't do too many requests at once
if requests_count > 16:
break
remoteData = backup_matrix.GetRemoteDataArray(
self.currentBackupID, blockNum)
remoteParity = backup_matrix.GetRemoteParityArray(
self.currentBackupID, blockNum)
localData = backup_matrix.GetLocalDataArray(
self.currentBackupID, blockNum)
localParity = backup_matrix.GetLocalParityArray(
self.currentBackupID, blockNum)
if supplierNum >= len(remoteData) or supplierNum >= len(remoteParity):
break
if supplierNum >= len(localData) or supplierNum >= len(localParity):
break
# if remote Data exist and is available because supplier is on-line,
# but we do not have it on hand - do request
if localData[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Data')
if remoteData[supplierNum] == 1:
if availableSuppliers[supplierNum]:
# if supplier is not alive - we can't request from him
if not io_throttle.HasPacketInRequestQueue(supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID):
requests_count += 1
else:
# count this packet as missing
self.missingPackets += 1
# also mark this guy as one who dont have any data - nor local nor remote
else:
# but if local Data already exists, but was not sent - do it now
if remoteData[supplierNum] != 1:
data_sender.A('new-data')
# same for Parity
if localParity[supplierNum] == 0:
PacketID = packetid.MakePacketID(
self.currentBackupID, blockNum, supplierNum, 'Parity')
if remoteParity[supplierNum] == 1:
if availableSuppliers[supplierNum]:
if not io_throttle.HasPacketInRequestQueue(
supplierID, PacketID):
customer, remotePath = packetid.SplitPacketID(PacketID)
filename = os.path.join(
settings.getLocalBackupsDir(),
customer,
remotePath,
)
if not os.path.exists(filename):
if io_throttle.QueueRequestFile(
self._file_received,
my_id.getLocalID(),
PacketID,
my_id.getLocalID(),
supplierID,
):
requests_count += 1
else:
self.missingPackets += 1
else:
# but if local Parity already exists, but was not sent - do it now
if remoteParity[supplierNum] != 1:
data_sender.A('new-data')
total_requests_count += requests_count
if total_requests_count > 0:
lg.out(8, 'backup_rebuilder._request_files : %d chunks requested' % total_requests_count)
self.automat('requests-sent', total_requests_count)
else:
if self.missingPackets:
lg.out(8, 'backup_rebuilder._request_files : found %d missing packets' % self.missingPackets)
self.automat('found-missing')
else:
lg.out(8, 'backup_rebuilder._request_files : nothing was requested')
self.automat('no-requests')
def A(self, event, *args, **kwargs):
from customer import fire_hire
from customer import data_sender
from customer import list_files_orator
from storage import backup_rebuilder
from storage import index_synchronizer
#---READY---
if self.state == 'READY':
if event == 'timer-5sec':
self.doOverallCheckUp(*args, **kwargs)
elif event == 'restart' or event == 'suppliers-changed' or (
event == 'instant' and self.RestartAgain):
self.state = 'FIRE_HIRE'
self.RestartAgain = False
self.doRememberSuppliers(*args, **kwargs)
fire_hire.A('restart')
#---LIST_FILES---
elif self.state == 'LIST_FILES':
if (event == 'list_files_orator.state' and args[0] == 'NO_FILES'):
self.state = 'READY'
elif (event == 'list_files_orator.state'
and args[0] == 'SAW_FILES'):
self.state = 'LIST_BACKUPS'
index_synchronizer.A('pull')
data_sender.A('restart')
self.doPrepareListBackups(*args, **kwargs)
elif event == 'restart':
self.RestartAgain = True
elif event == 'suppliers-changed':
self.state = 'READY'
self.RestartAgain = True
#---LIST_BACKUPS---
elif self.state == 'LIST_BACKUPS':
if event == 'list-backups-done':
self.state = 'REBUILDING'
backup_rebuilder.A('start')
elif event == 'restart':
self.RestartAgain = True
elif event == 'suppliers-changed':
self.state = 'READY'
self.RestartAgain = True
elif event == 'restart':
self.state = 'FIRE_HIRE'
fire_hire.A('restart')
#---REBUILDING---
elif self.state == 'REBUILDING':
if (event == 'backup_rebuilder.state'
and args[0] in ['DONE', 'STOPPED']):
self.state = 'READY'
self.doCleanUpBackups(*args, **kwargs)
data_sender.A('restart')
elif event == 'restart' or event == 'suppliers-changed':
self.state = 'FIRE_HIRE'
backup_rebuilder.SetStoppedFlag()
fire_hire.A('restart')
#---FIRE_HIRE---
elif self.state == 'FIRE_HIRE':
if event == 'suppliers-changed' and self.isSuppliersNumberChanged(
*args, **kwargs):
self.state = 'LIST_FILES'
self.doDeleteAllBackups(*args, **kwargs)
self.doRememberSuppliers(*args, **kwargs)
list_files_orator.A('need-files')
elif event == 'fire-hire-finished':
self.state = 'LIST_FILES'
list_files_orator.A('need-files')
elif event == 'suppliers-changed' and not self.isSuppliersNumberChanged(
*args, **kwargs):
self.state = 'LIST_FILES'
self.doUpdateSuppliers(*args, **kwargs)
self.doRememberSuppliers(*args, **kwargs)
list_files_orator.A('need-files')
elif event == 'restart':
self.RestartAgain = True
#---AT_STARTUP---
elif self.state == 'AT_STARTUP':
if event == 'init':
self.state = 'READY'
self.RestartAgain = False
return None
def A(self, event, *args, **kwargs):
from customer import data_sender
#---AT_STARTUP---
if self.state == 'AT_STARTUP':
if event == 'start':
self.state = 'READ'
self.doInit(*args, **kwargs)
self.doFirstBlock(*args, **kwargs)
#---READ---
elif self.state == 'READ':
if event == 'read-success' and not self.isReadingNow(
*args, **kwargs) and (self.isBlockReady(*args, **kwargs)
or self.isEOF(*args, **kwargs)):
self.state = 'ENCRYPT'
self.doEncryptBlock(*args, **kwargs)
elif event == 'fail' or (
(event == 'read-success' or event == 'timer-001sec')
and self.isAborted(*args, **kwargs)):
self.state = 'ABORTED'
self.doClose(*args, **kwargs)
self.doReport(*args, **kwargs)
self.doDestroyMe(*args, **kwargs)
elif (event == 'read-success'
or event == 'timer-001sec') and not self.isAborted(
*args, **kwargs) and self.isPipeReady(
*args, **kwargs) and not self.isEOF(
*args, **kwargs) and not self.isReadingNow(
*args, **kwargs) and not self.isBlockReady(
*args, **kwargs):
self.doRead(*args, **kwargs)
elif event == 'block-raid-done' and not self.isAborted(
*args, **kwargs):
self.doPopBlock(*args, **kwargs)
self.doBlockReport(*args, **kwargs)
data_sender.A('new-data')
#---RAID---
elif self.state == 'RAID':
if event == 'block-raid-done' and not self.isMoreBlocks(
*args, **kwargs) and not self.isAborted(*args, **kwargs):
self.state = 'DONE'
self.doPopBlock(*args, **kwargs)
self.doBlockReport(*args, **kwargs)
data_sender.A('new-data')
self.doClose(*args, **kwargs)
self.doReport(*args, **kwargs)
self.doDestroyMe(*args, **kwargs)
elif event == 'block-raid-started' and not self.isEOF(
*args, **kwargs) and not self.isAborted(*args, **kwargs):
self.state = 'READ'
self.doNextBlock(*args, **kwargs)
self.doRead(*args, **kwargs)
elif event == 'block-raid-done' and self.isMoreBlocks(
*args, **kwargs) and not self.isAborted(*args, **kwargs):
self.doPopBlock(*args, **kwargs)
self.doBlockReport(*args, **kwargs)
data_sender.A('new-data')
elif event == 'fail' or (
(event == 'timer-01sec' or event == 'block-raid-done' or event
== 'block-raid-started') and self.isAborted(*args, **kwargs)):
self.state = 'ABORTED'
self.doClose(*args, **kwargs)
self.doReport(*args, **kwargs)
self.doDestroyMe(*args, **kwargs)
#---DONE---
elif self.state == 'DONE':
pass
#---ABORTED---
elif self.state == 'ABORTED':
pass
#---ENCRYPT---
elif self.state == 'ENCRYPT':
if event == 'block-encrypted':
self.state = 'RAID'
self.doBlockPushAndRaid(*args, **kwargs)
elif event == 'fail':
self.state = 'ABORTED'
self.doClose(*args, **kwargs)
self.doReport(*args, **kwargs)
self.doDestroyMe(*args, **kwargs)
elif event == 'block-raid-done' and not self.isAborted(
*args, **kwargs):
self.doPopBlock(*args, **kwargs)
self.doBlockReport(*args, **kwargs)
data_sender.A('new-data')
return None
