def doSavePacket(self, *args, **kwargs): """ Action method. """ if not args or not args[0]: raise Exception('no input found') NewPacket, PacketID = args[0] glob_path = global_id.ParseGlobalID(PacketID, detect_version=True) packetID = global_id.CanonicalID(PacketID) customer_id, _, _, _, SupplierNumber, dataORparity = packetid.SplitFull(packetID) if dataORparity == 'Data': self.OnHandData[SupplierNumber] = True elif dataORparity == 'Parity': self.OnHandParity[SupplierNumber] = True if not NewPacket: lg.warn('packet %r already exists locally' % packetID) return filename = os.path.join(settings.getLocalBackupsDir(), customer_id, glob_path['path']) dirpath = os.path.dirname(filename) if not os.path.exists(dirpath): try: bpio._dirs_make(dirpath) except: lg.exc() # either way the payload of packet is saved if not bpio.WriteBinaryFile(filename, NewPacket.Payload): lg.err("unable to write to %s" % filename) return if self.packetInCallback is not None: self.packetInCallback(self.backup_id, NewPacket) if _Debug: lg.out(_DebugLevel, "restore_worker.doSavePacket %s saved to %s" % (packetID, filename))
def doSavePacket(self, NewPacket): glob_path = global_id.ParseGlobalID(NewPacket.PacketID, detect_version=True) packetID = global_id.CanonicalID(NewPacket.PacketID) customer_id, _, _, _, SupplierNumber, dataORparity = packetid.SplitFull( packetID) if dataORparity == 'Data': self.OnHandData[SupplierNumber] = True elif NewPacket.DataOrParity() == 'Parity': self.OnHandParity[SupplierNumber] = True filename = os.path.join(settings.getLocalBackupsDir(), customer_id, glob_path['path']) dirpath = os.path.dirname(filename) if not os.path.exists(dirpath): try: bpio._dirs_make(dirpath) except: lg.exc() # either way the payload of packet is saved if not bpio.WriteFile(filename, NewPacket.Payload): lg.warn("unable to write to %s" % filename) return if self.packetInCallback is not None: self.packetInCallback(self.BackupID, NewPacket) lg.out(6, "restore.doSavePacket %s saved to %s" % (packetID, filename))
def doPullPacket(self, event, *args, **kwargs): """ Action method. """ newpacket = kwargs['newpacket'] _, _, _, block_num, _, _ = packetid.SplitFull(newpacket.PacketID) if block_num not in self.packets_out: raise Exception('unregistered block number received') if newpacket.PacketID not in self.packets_out[block_num]: raise Exception('unregistered packet ID received') if event == 'ack': self.packets_out[block_num][newpacket.PacketID] = True else: self.packets_out[block_num][newpacket.PacketID] = False if _Debug: lg.args(_DebugLevel, event=event, block_num=block_num, packet_id=newpacket.PacketID, packets_out=self.packets_out)
def cb(result, realpath, subpath, name): if not os.access(realpath, os.R_OK): return False if os.path.isfile(realpath): try: filesz = os.path.getsize(realpath) except: filesz = -1 result.write('F%s %d\n' % (subpath, filesz)) return False if not packetid.IsCanonicalVersion(name): result.write('D%s\n' % subpath) return True maxBlock = -1 versionSize = {} dataBlocks = {} parityBlocks = {} dataMissing = {} parityMissing = {} for filename in os.listdir(realpath): packetID = subpath + '/' + filename pth = os.path.join(realpath, filename) try: filesz = os.path.getsize(pth) except: filesz = -1 if os.path.isdir(pth): result.write('D%s\n' % packetID) continue if not packetid.Valid(packetID): result.write('F%s %d\n' % (packetID, filesz)) continue customer, pathID, versionName, blockNum, supplierNum, dataORparity = packetid.SplitFull( packetID) if None in [ pathID, versionName, blockNum, supplierNum, dataORparity ]: result.write('F%s %d\n' % (packetID, filesz)) continue if dataORparity != 'Data' and dataORparity != 'Parity': result.write('F%s %d\n' % (packetID, filesz)) continue if maxBlock < blockNum: maxBlock = blockNum if supplierNum not in versionSize: versionSize[supplierNum] = 0 if supplierNum not in dataBlocks: dataBlocks[supplierNum] = {} if supplierNum not in parityBlocks: parityBlocks[supplierNum] = {} if dataORparity == 'Data': dataBlocks[supplierNum][blockNum] = filesz elif dataORparity == 'Parity': parityBlocks[supplierNum][blockNum] = filesz for supplierNum in versionSize.keys(): dataMissing[supplierNum] = set(range(maxBlock + 1)) parityMissing[supplierNum] = set(range(maxBlock + 1)) for blockNum in range(maxBlock + 1): if blockNum in list(dataBlocks[supplierNum].keys()): versionSize[supplierNum] += dataBlocks[supplierNum][ blockNum] dataMissing[supplierNum].discard(blockNum) if blockNum in list(parityBlocks[supplierNum].keys()): versionSize[supplierNum] += parityBlocks[supplierNum][ blockNum] parityMissing[supplierNum].discard(blockNum) suppliers = set(list(dataBlocks.keys()) + list(parityBlocks.keys())) for supplierNum in suppliers: versionString = '%s %d 0-%d %d' % (subpath, supplierNum, maxBlock, versionSize[supplierNum]) if len(dataMissing[supplierNum]) > 0 or len( parityMissing[supplierNum]) > 0: versionString += ' missing' if len(dataMissing[supplierNum]) > 0: versionString += ' Data:' + (','.join( map(str, dataMissing[supplierNum]))) if len(parityMissing[supplierNum]) > 0: versionString += ' Parity:' + (','.join( map(str, parityMissing[supplierNum]))) result.write('V%s\n' % versionString) del dataBlocks del parityBlocks del dataMissing del parityMissing return False