def doSavePacket(self, *args, **kwargs):
"""
Action method.
"""
if not args or not args[0]:
raise Exception('no input found')
NewPacket, PacketID = args[0]
glob_path = global_id.ParseGlobalID(PacketID, detect_version=True)
packetID = global_id.CanonicalID(PacketID)
customer_id, _, _, _, SupplierNumber, dataORparity = packetid.SplitFull(packetID)
if dataORparity == 'Data':
self.OnHandData[SupplierNumber] = True
elif dataORparity == 'Parity':
self.OnHandParity[SupplierNumber] = True
if not NewPacket:
lg.warn('packet %r already exists locally' % packetID)
return
filename = os.path.join(settings.getLocalBackupsDir(), customer_id, glob_path['path'])
dirpath = os.path.dirname(filename)
if not os.path.exists(dirpath):
try:
bpio._dirs_make(dirpath)
except:
lg.exc()
# either way the payload of packet is saved
if not bpio.WriteBinaryFile(filename, NewPacket.Payload):
lg.err("unable to write to %s" % filename)
return
if self.packetInCallback is not None:
self.packetInCallback(self.backup_id, NewPacket)
if _Debug:
lg.out(_DebugLevel, "restore_worker.doSavePacket %s saved to %s" % (packetID, filename))
def doSavePacket(self, NewPacket):
glob_path = global_id.ParseGlobalID(NewPacket.PacketID,
detect_version=True)
packetID = global_id.CanonicalID(NewPacket.PacketID)
customer_id, _, _, _, SupplierNumber, dataORparity = packetid.SplitFull(
packetID)
if dataORparity == 'Data':
self.OnHandData[SupplierNumber] = True
elif NewPacket.DataOrParity() == 'Parity':
self.OnHandParity[SupplierNumber] = True
filename = os.path.join(settings.getLocalBackupsDir(), customer_id,
glob_path['path'])
dirpath = os.path.dirname(filename)
if not os.path.exists(dirpath):
try:
bpio._dirs_make(dirpath)
except:
lg.exc()
# either way the payload of packet is saved
if not bpio.WriteFile(filename, NewPacket.Payload):
lg.warn("unable to write to %s" % filename)
return
if self.packetInCallback is not None:
self.packetInCallback(self.BackupID, NewPacket)
lg.out(6, "restore.doSavePacket %s saved to %s" % (packetID, filename))
def doPullPacket(self, event, *args, **kwargs):
"""
Action method.
"""
newpacket = kwargs['newpacket']
_, _, _, block_num, _, _ = packetid.SplitFull(newpacket.PacketID)
if block_num not in self.packets_out:
raise Exception('unregistered block number received')
if newpacket.PacketID not in self.packets_out[block_num]:
raise Exception('unregistered packet ID received')
if event == 'ack':
self.packets_out[block_num][newpacket.PacketID] = True
else:
self.packets_out[block_num][newpacket.PacketID] = False
if _Debug:
lg.args(_DebugLevel,
event=event,
block_num=block_num,
packet_id=newpacket.PacketID,
packets_out=self.packets_out)
def cb(result, realpath, subpath, name):
if not os.access(realpath, os.R_OK):
return False
if os.path.isfile(realpath):
try:
filesz = os.path.getsize(realpath)
except:
filesz = -1
result.write('F%s %d\n' % (subpath, filesz))
return False
if not packetid.IsCanonicalVersion(name):
result.write('D%s\n' % subpath)
return True
maxBlock = -1
versionSize = {}
dataBlocks = {}
parityBlocks = {}
dataMissing = {}
parityMissing = {}
for filename in os.listdir(realpath):
packetID = subpath + '/' + filename
pth = os.path.join(realpath, filename)
try:
filesz = os.path.getsize(pth)
except:
filesz = -1
if os.path.isdir(pth):
result.write('D%s\n' % packetID)
continue
if not packetid.Valid(packetID):
result.write('F%s %d\n' % (packetID, filesz))
continue
customer, pathID, versionName, blockNum, supplierNum, dataORparity = packetid.SplitFull(
packetID)
if None in [
pathID, versionName, blockNum, supplierNum, dataORparity
]:
result.write('F%s %d\n' % (packetID, filesz))
continue
if dataORparity != 'Data' and dataORparity != 'Parity':
result.write('F%s %d\n' % (packetID, filesz))
continue
if maxBlock < blockNum:
maxBlock = blockNum
if supplierNum not in versionSize:
versionSize[supplierNum] = 0
if supplierNum not in dataBlocks:
dataBlocks[supplierNum] = {}
if supplierNum not in parityBlocks:
parityBlocks[supplierNum] = {}
if dataORparity == 'Data':
dataBlocks[supplierNum][blockNum] = filesz
elif dataORparity == 'Parity':
parityBlocks[supplierNum][blockNum] = filesz
for supplierNum in versionSize.keys():
dataMissing[supplierNum] = set(range(maxBlock + 1))
parityMissing[supplierNum] = set(range(maxBlock + 1))
for blockNum in range(maxBlock + 1):
if blockNum in list(dataBlocks[supplierNum].keys()):
versionSize[supplierNum] += dataBlocks[supplierNum][
blockNum]
dataMissing[supplierNum].discard(blockNum)
if blockNum in list(parityBlocks[supplierNum].keys()):
versionSize[supplierNum] += parityBlocks[supplierNum][
blockNum]
parityMissing[supplierNum].discard(blockNum)
suppliers = set(list(dataBlocks.keys()) + list(parityBlocks.keys()))
for supplierNum in suppliers:
versionString = '%s %d 0-%d %d' % (subpath, supplierNum, maxBlock,
versionSize[supplierNum])
if len(dataMissing[supplierNum]) > 0 or len(
parityMissing[supplierNum]) > 0:
versionString += ' missing'
if len(dataMissing[supplierNum]) > 0:
versionString += ' Data:' + (','.join(
map(str, dataMissing[supplierNum])))
if len(parityMissing[supplierNum]) > 0:
versionString += ' Parity:' + (','.join(
map(str, parityMissing[supplierNum])))
result.write('V%s\n' % versionString)
del dataBlocks
del parityBlocks
del dataMissing
del parityMissing
return False
