def get_record(self, recno): if not self.mode: raise TarDB.FileError('get_record: not opened: %r' % self) try: (name, offset) = self._catalog.get(recno) except Catalog.InvalidRecord: raise TarDB.InvalidRecord(recno) tarfp = self._get_tarfile(name) tarfp.seek(offset) buf = tarfp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: raise TarDB.Corrupted( 'get_record: premature eof in info block: %r, recno=%d, info_offset=%d' % (self, recno, offset)) try: info = TarInfo.frombuf(buf) except ValueError: raise TarDB.Corrupted( 'get_record: tar record corrupted: %r, recno=%d, offset=%d' % (self, recno, offset)) data = tarfp.read(info.size) if len(data) != info.size: raise TarDB.Corrupted( 'get_record: premature eof in data block: %r, recno=%d, info_offset=%d' % (self, recno, offset)) return (info, data)
def generate_catalog(args, recsize, out=stdout, verbose=0): out.write(''.join(str(i % 10) for i in xrange(1, recsize)) + '\n') args.sort() recno = 0 errno = 0 for fname in args: if verbose: print >> stderr, 'reading: %r...' % fname name = basename(fname) if name.endswith('.tar'): name = name[:-4] if recsize < (len(name) + 9): print >> stderr, 'tarfile %r does not fit to the catalog record size.' % fname errno = 1 continue name += ' ' * (recsize - 9 - len(name)) missing = 'x' * (recsize - 1) tarfp = file(fname, 'rb') while 1: offset = tarfp.tell() buf = tarfp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: break try: info = TarInfo.frombuf(buf) except ValueError: print >> stderr, '%r: tar record corrputed at offset=%d' % ( fname, offset) errno = 1 break size = ((info.size + BLOCKSIZE - 1) / BLOCKSIZE) * BLOCKSIZE data = tarfp.read(size) if len(data) != size: print >> stderr, '%r: premature eof at offset=%d' % (fname, offset) errno = 1 break recid = int(info.name[:8], 16) if recid < recno: print >> stderr, '%r: duplicated recno: %d at offset=%d' % ( fname, recid, offset) errno = 1 continue if recno < recid: print >> stderr, '%r: missing recno: %d-%d' % (fname, recno, recid - 1) for _ in xrange(recno, recid): out.write(missing + '\n') recno = recid errno = 1 out.write('%08x%s\n' % (offset, name)) recno += 1 tarfp.close() return errno
def extract_tar_entry(self): if self.append: return True buf = self.read(BLOCKSIZE) if buf == NUL * BLOCKSIZE: return False self.inf = TarInfo.frombuf(buf) if self.mode == self.MODE_RANGE: self.inf.size = min(self.req_size - self.state['consumed'], self.inf.size) return True
def generate_labelidx(args, prefix, verbose=0): import re, struct pat = re.compile(r'([0-9a-f]{8})\.(.*)') valid_labels = re.compile(r'[0-9a-zA-Z]') labelmap = {} errno = 0 for fname in args: if verbose: print >> stderr, 'reading: %r...' % fname tarfp = file(fname, 'rb') while 1: offset = tarfp.tell() buf = tarfp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: break try: info = TarInfo.frombuf(buf) except ValueError: print >> stderr, '%r: tar record corrputed at offset=%d' % ( fname, offset) errno = 1 break size = ((info.size + BLOCKSIZE - 1) / BLOCKSIZE) * BLOCKSIZE data = tarfp.read(size) if len(data) != size: print >> stderr, '%r: premature eof at offset=%d' % (fname, offset) errno = 1 break m = pat.match(info.name) if not m: print >> stderr, '%r: invalid name %r at offset=%d' % ( fname, info.name, offset) errno = 1 continue (recno, labels) = m.groups() recno = int(recno, 16) for label in valid_labels.findall(labels): if label not in labelmap: labelmap[label] = [] labelmap[label].append(recno) tarfp.close() # for (label, recnos) in labelmap.iteritems(): fname = '%s_%02x' % (prefix, ord(label)) recnos.sort(reverse=True) data = struct.pack('>%di' % len(recnos), *recnos) fp = file(fname, 'wb') fp.write(data) fp.close() return errno
def generate_catalog(args, recsize, out=stdout, verbose=0): out.write(''.join( str(i % 10) for i in xrange(1,recsize) ) + '\n') args.sort() recno = 0 errno = 0 for fname in args: if verbose: print >>stderr, 'reading: %r...' % fname name = basename(fname) if name.endswith('.tar'): name = name[:-4] if recsize < (len(name) + 9): print >>stderr, 'tarfile %r does not fit to the catalog record size.' % fname errno = 1 continue name += ' '*(recsize-9-len(name)) missing = 'x'*(recsize-1) tarfp = file(fname, 'rb') while 1: offset = tarfp.tell() buf = tarfp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: break try: info = TarInfo.frombuf(buf) except ValueError: print >>stderr, '%r: tar record corrputed at offset=%d' % (fname, offset) errno = 1 break size = ((info.size + BLOCKSIZE-1) / BLOCKSIZE) * BLOCKSIZE data = tarfp.read(size) if len(data) != size: print >>stderr, '%r: premature eof at offset=%d' % (fname, offset) errno = 1 break recid = int(info.name[:8], 16) if recid < recno: print >>stderr, '%r: duplicated recno: %d at offset=%d' % (fname, recid, offset) errno = 1 continue if recno < recid: print >>stderr, '%r: missing recno: %d-%d' % (fname, recno, recid-1) for _ in xrange(recno, recid): out.write(missing+'\n') recno = recid errno = 1 out.write('%08x%s\n' % (offset, name)) recno += 1 tarfp.close() return errno
def get_info(self, recno): if not self.mode: raise TarDB.FileError('get_info: not opened: %r' % self) try: (name, offset) = self._catalog.get(recno) except Catalog.InvalidRecord: raise TarDB.InvalidRecord(recno) tarfp = self._get_tarfile(name) tarfp.seek(offset) buf = tarfp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: raise TarDB.Corrupted('get_info: premature eof in info block: %r, recno=%d, info_offset=%d' % (self, recno, offset)) try: return TarInfo.frombuf(buf) except ValueError: raise TarDB.Corrupted('get_info: tar record corrupted: %r, recno=%d, info_offset=%d' % (self, recno, offset))
def get_recinfo(self, recno, data=False): (idx, offset) = self._get_catent(recno) if self._ridx != idx: if self._rtar is not None: self._rtar.close() assert idx in self._files path = os.path.join(self.basedir, self._files[idx]) self._rtar = open(path, mode='rb') self._ridx = idx self._rtar.seek(offset) buf = self._rtar.read(BLOCKSIZE) info = TarInfo.frombuf(buf, 'utf-8', 'ignore') if data: b = self._rtar.read(info.size) else: b = None return (info, b)
def generate_labelidx(args, prefix, verbose=0): import re, struct pat = re.compile(r'([0-9a-f]{8})\.(.*)') valid_labels = re.compile(r'[0-9a-zA-Z]') labelmap = {} errno = 0 for fname in args: if verbose: print >>stderr, 'reading: %r...' % fname tarfp = file(fname, 'rb') while 1: offset = tarfp.tell() buf = tarfp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: break try: info = TarInfo.frombuf(buf) except ValueError: print >>stderr, '%r: tar record corrputed at offset=%d' % (fname, offset) errno = 1 break size = ((info.size + BLOCKSIZE-1) / BLOCKSIZE) * BLOCKSIZE data = tarfp.read(size) if len(data) != size: print >>stderr, '%r: premature eof at offset=%d' % (fname, offset) errno = 1 break m = pat.match(info.name) if not m: print >>stderr, '%r: invalid name %r at offset=%d' % (fname, info.name, offset) errno = 1 continue (recno, labels) = m.groups() recno = int(recno, 16) for label in valid_labels.findall(labels): if label not in labelmap: labelmap[label] = [] labelmap[label].append(recno) tarfp.close() # for (label,recnos) in labelmap.iteritems(): fname = '%s_%02x' % (prefix, ord(label)) recnos.sort(reverse=True) data = struct.pack('>%di' % len(recnos), *recnos) fp = file(fname, 'wb') fp.write(data) fp.close() return errno
def extract_tar_entry(self): if self.append: return True buf = self.read(BLOCKSIZE) if buf == NUL * BLOCKSIZE: return False self.inf = TarInfo.frombuf(buf) if self.mode == self.MODE_RANGE: self.inf.size = min(self.req_size - self.state['consumed'], self.inf.size) if 'manifest' in self.cur_state and self.cur_state['manifest']: for entry in self.cur_state['manifest']: if entry['name'] == self.inf.name: self.cur_state['entry'] = entry return True
def _get_catalog(self): for fname in os.listdir(self.basedir): try: idx = self.name2idx(fname) except ValueError: continue try: fp = file(os.path.join(self.basedir, fname), 'rb') except IOError, e: raise TarDB.FileError(e) while 1: offset = fp.tell() buf = fp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: break info = TarInfo.frombuf(buf) size = ((info.size + BLOCKSIZE - 1) / BLOCKSIZE) * BLOCKSIZE if len(fp.read(size)) != size: break yield (idx, offset) fp.close()
def _get_catalog(self): for fname in os.listdir(self.basedir): try: idx = self.name2idx(fname) except ValueError: continue try: fp = file(os.path.join(self.basedir, fname), 'rb') except IOError, e: raise TarDB.FileError(e) while 1: offset = fp.tell() buf = fp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: break info = TarInfo.frombuf(buf) size = ((info.size + BLOCKSIZE-1) / BLOCKSIZE) * BLOCKSIZE if len(fp.read(size)) != size: break yield (idx, offset) fp.close()
def get_record(self, recno): if not self._mode: raise TarDB.FileError('not open: %r' % self) (idx, offset) = self.get_filepos(recno) fp = self._get_fp(idx) try: fp.open() except FileLock.Failed, e: raise TarDB.LockError(e) fp.seek(offset) buf = fp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: raise TarDB.Corrupted( 'get_record: premature eof info: %r, recno=%d, info_offset=%d' % (self, recno, offset)) try: info = TarInfo.frombuf(buf) except ValueError: raise TarDB.Corrupted( 'get_record: record corrupted: %r, recno=%d, offset=%d' % (self, recno, offset)) data = fp.read(info.size) if len(data) != info.size: raise TarDB.Corrupted( 'get_record: premature eof data: %r, recno=%d, info_offset=%d' % (self, recno, offset)) fp.close() return (info, data) def get_info(self, recno): if not self._mode: raise TarDB.FileError('not open: %r' % self) (idx, offset) = self.get_filepos(recno)
def get_record(self, recno): if not self._mode: raise TarDB.FileError('not open: %r' % self) (idx, offset) = self.get_filepos(recno) fp = self._get_fp(idx) try: fp.open() except FileLock.Failed, e: raise TarDB.LockError(e) fp.seek(offset) buf = fp.read(BLOCKSIZE) if len(buf) != BLOCKSIZE: raise TarDB.Corrupted('get_record: premature eof info: %r, recno=%d, info_offset=%d' % (self, recno, offset)) try: info = TarInfo.frombuf(buf) except ValueError: raise TarDB.Corrupted('get_record: record corrupted: %r, recno=%d, offset=%d' % (self, recno, offset)) data = fp.read(info.size) if len(data) != info.size: raise TarDB.Corrupted('get_record: premature eof data: %r, recno=%d, info_offset=%d' % (self, recno, offset)) fp.close() return (info, data) def get_info(self, recno): if not self._mode: raise TarDB.FileError('not open: %r' % self) (idx, offset) = self.get_filepos(recno) fp = self._get_fp(idx) try: