def _testCs(repos, troves, idxLength, fileCount): job = [(x.getName(), (None, None), (x.getVersion(), x.getFlavor()), True) for x in troves] repos.createChangeSetFile(job, self.workDir + '/foo.ccs') fc = filecontainer.FileContainer( util.ExtendedFile(self.workDir + '/foo.ccs', "r", buffering=False)) info = fc.getNextFile() assert (info[0] == 'CONARYCHANGESET') info = fc.getNextFile() while info is not None: assert (len(info[0]) == idxLength) fileCount -= 1 if 'ptr' in info[1]: s = info[2].read() s = gzip.GzipFile(None, "r", fileobj=StringIO(s)).read() assert (len(s) == idxLength) info = fc.getNextFile() assert (fileCount == 0)
def testSeekableNestedFile(self): (fd, name) = tempfile.mkstemp() f = util.ExtendedFile(name, "w++", buffering = False) os.close(fd) os.unlink(name) s = [ "hello world", "foo bar bang" ] fs = [] f.write(s[0]) fs.append(util.SeekableNestedFile(f, len(s[0]), 0)) fs.append(util.SeekableNestedFile(f, len(s[1]))) f.write(s[1]) assert(fs[0].read() == s[0]) assert(fs[1].read() == s[1]) assert(fs[0].read() == "") assert(fs[1].read() == "") assert(fs[0].pread(offset = 0) == s[0]) assert(fs[0].read() == "") fs[0].seek(0) assert(fs[1].read() == "") assert(fs[0].read() == s[0]) assert(fs[0].read() == "") fs[0].seek(5) assert(fs[0].read() == s[0][5:]) fs[0].seek(5 - len(s[0]), 2) assert(fs[0].read() == s[0][5:]) fs[0].seek(5) fs[0].seek(2, 1) assert(fs[0].read() == s[0][7:])
def testSeekableNestedFileNested(self): # Nested nested files s = "0123456789" (fd, name) = tempfile.mkstemp() f = util.ExtendedFile(name, "w+", buffering = False) os.close(fd) os.unlink(name) f.write(s) # Start from the second byte, make sure pread works f1 = util.SeekableNestedFile(f, 9, 1) first = f1.pread(1, 0) self.assertEqual(first, '1') # Create nested files within the first nested file f21 = util.SeekableNestedFile(f1, 5, 1) # Make sure pread, read, tell all work as expected first = f21.pread(1, 0) self.assertEqual(first, '2') self.assertEqual(f21.read(), '23456') self.assertEqual(f21.tell(), 5) f31 = util.SeekableNestedFile(f21, 3, 4) self.assertEqual(f31.read(), '6')
def testVerifySig(self): rpmName = 'tmpwatch-2.9.7-1.1.el5.2.x86_64.rpm' rpmPath = os.path.join(self.archiveDir, rpmName) fileObj = file(rpmPath) header = rpmhelper.readHeader(fileObj) from conary.lib import openpgpfile sig = openpgpfile.readSignature(header[rpmhelper.SIG_GPG]) fileObj.seek(0) rpmhelper.readSignatureHeader(fileObj) k = openpgpfile.getKeyFromString( 'E8562897', openpgpfile.parseAsciiArmorKey(pgpKeyCentos)) rpmhelper.verifySignatures(fileObj, [k]) # Similar deal, fileObj is an ExtendedFile fileObj = util.ExtendedFile(rpmPath, buffering=False) rpmhelper.verifySignatures(fileObj, [k]) # Finally, StringIO fileObj.seek(0) fileObj = StringIO.StringIO(fileObj.read()) rpmhelper.verifySignatures(fileObj, [k]) # Replace last byte fileObj = StringIO.StringIO(fileObj.getvalue()[:-1]) fileObj.seek(0, 2) fileObj.write("\xff") fileObj.seek(0) e = self.assertRaises(rpmhelper.MD5SignatureError, rpmhelper.verifySignatures, fileObj, [k]) self.assertEqual( str(e), 'The MD5 digest fails to verify: ' 'expected 6cc7c546c3a5de90bb272b11be2f3d67, got 744d88f4164ec2974b49839a69ea589d' )
def csContents(repos, job): p = os.path.join(self.workDir, 'foo.ccs') repos.createChangeSetFile(job, p) fc = filecontainer.FileContainer( util.ExtendedFile(p, buffering = False)) l = [] info = fc.getNextFile() while info: l.append(info) info = fc.getNextFile() return l
def testExtendedFile(self): fd, fn = tempfile.mkstemp() try: os.write(fd, "hello world") os.close(fd) f = util.ExtendedFile(fn, buffering=False) assert(f.read(5) == 'hello') assert(f.pread(5, 6) == 'world') assert(f.tell() == 5) finally: os.unlink(fn)
def test07_BadRpmCapsule(self): cmp = self.addRPMComponent("ghost:rpm=1.0", 'epoch-1.0-1.i386.rpm') repos = self.openRepository() orig = self.workDir + '/ghost.ccs' modified = self.workDir + '/ghost-new.ccs' repos.createChangeSetFile([ (cmp.getName(), (None, None), cmp.getNameVersionFlavor()[1:], True) ], orig) fc = filecontainer.FileContainer( util.ExtendedFile(orig, buffering = False)) newFc = filecontainer.FileContainer( util.ExtendedFile(modified, "w", buffering = False)) # CONARYCHANGESET (name, tag, contents) = fc.getNextFile() newFc.addFile(name, filecontents.FromFile(contents), tag, precompressed = True) # the RPM (name, tag, contents) = fc.getNextFile() contents = filecontents.FromString("busted!") newFc.addFile(name, contents, tag) cs = changeset.ChangeSetFromFile(modified) # this throws away the output about the install failing self.assertRaises(files.Sha1Exception, self.captureOutput, self.updatePkg, self.rootDir, cs)
def _produceChangeset(self, items): readNestedFile = proxy.ChangesetFileReader.readNestedFile for path, isChangeset, preserveFile in items: if isChangeset: csFile = util.ExtendedFile(path, 'rb', buffering=False) changeSet = filecontainer.FileContainer(csFile) for data in changeSet.dumpIter(readNestedFile, args=(self.contentsStore, )): yield data del changeSet else: fobj = open(path, 'rb') for data in util.iterFileChunks(fobj): yield data fobj.close() if not preserveFile: os.unlink(path)
def testMergedConfigOrder(self): # Make sure that config files from absolute change sets are merged # correctly relative to config files from relative ones. t1 = self.addComponent('test:runtime', '1.0-1-1', fileContents = [ ("/etc/cfg", "contents1\n") ] ) # This filename is magically designed to have it's pathId before # the pathId for /etc/cfg (the pathId in addComponent is a # simple md5 of the path) o = self.addComponent('other:runtime', '1.0-1-1', fileContents = [ ("/etc/one", "something") ] ) repos = self.openRepository() (fd, path) = tempfile.mkstemp() os.close(fd) repos.createChangeSetFile( [ (o.getName(), (None, None), (o.getVersion(), o.getFlavor()), False), (t1.getName(), (None, None), (t1.getVersion(), t1.getFlavor()), True) ], path) f = util.ExtendedFile(path, "r", buffering = False) os.unlink(path) fc = filecontainer.FileContainer(f) # first comes the set of troveCs objects (name, tag, size) = fc.getNextFile() assert(name == 'CONARYCHANGESET') # next is the diff (name, tag, size) = fc.getNextFile() assert(name[0:16] == sha1helper.md5String("/etc/one")) # and then the config file (name, tag, size) = fc.getNextFile() assert(name[0:16] == sha1helper.md5String("/etc/cfg")) # and that's it rc = fc.getNextFile() assert(rc is None)
def testChangeSetDumpOffset(self): """Stress test offset arg to dumpIter""" # Make a changeset with one regular file cs = changeset.ChangeSet() pathId = '0' * 16 fileId = '0' * 20 contents = 'contents' store = datastore.FlatDataStore(self.workDir) sha1 = sha1helper.sha1String(contents) store.addFile(StringIO(contents), sha1) rawFile = store.openRawFile(sha1) rawSize = os.fstat(rawFile.fileno()).st_size contObj = filecontents.CompressedFromDataStore(store, sha1) cs.addFileContents(pathId, fileId, changeset.ChangedFileTypes.file, contObj, cfgFile=False, compressed=True) # Test dumping a fully populated changeset with every possible resume # point path = os.path.join(self.workDir, 'full.ccs') size = cs.writeToFile(path) expected = open(path).read() self.assertEqual(len(expected), size) fc = filecontainer.FileContainer( util.ExtendedFile(path, 'r', buffering=False)) def noop(name, tag, size, subfile): assert tag[2:] != changeset.ChangedFileTypes.refr[4:] return tag, size, subfile for offset in range(size + 1): fc.reset() actual = ''.join(fc.dumpIter(noop, (), offset)) self.assertEqual(actual, expected[offset:]) # Test dumping a changeset with contents stripped out path = os.path.join(self.workDir, 'stubby.ccs') size2 = cs.writeToFile(path, withReferences=True) self.assertEqual(size2, size) fc = filecontainer.FileContainer( util.ExtendedFile(path, 'r', buffering=False)) expect_reference = '%s %d' % (sha1.encode('hex'), rawSize) def addfile(name, tag, size, subfile, dummy): self.assertEqual(dummy, 'dummy') if name == 'CONARYCHANGESET': return tag, size, subfile elif name == pathId + fileId: self.assertEqual(tag[2:], changeset.ChangedFileTypes.refr[4:]) self.assertEqual(subfile.read(), expect_reference) tag = tag[0:2] + changeset.ChangedFileTypes.file[4:] rawFile.seek(0) return tag, rawSize, rawFile else: assert False for offset in range(size + 1): fc.reset() actual = ''.join(fc.dumpIter(addfile, ('dummy', ), offset)) self.assertEqual(actual, expected[offset:])
def test(self): count = fileCount() # let's make sure we can't open an arbitrary file as a container f = util.ExtendedFile("/bin/ls", "r", buffering=False) self.assertRaises(filecontainer.BadContainer, FileContainer, f) f.close() if (count != fileCount()): raise AssertionError, "too many files are open %s" % count # create a new container f = util.ExtendedFile(self.fn, "w+", buffering=False) c = FileContainer(f) c.close() data = [] tags = [] names = [] c = FileContainer(f) self.assertRaises(AssertionError, c.addFile, "name", FromString("data"), "tag") c.close() os.unlink(self.fn) f = util.ExtendedFile(self.fn, "w+", buffering=False) c = FileContainer(f) data.append("contents of file1") tags.append("extra data") names.append("file1") c.addFile(names[0], FromString(data[0]), tags[0]) data.append("file2 gets some contents") tags.append("tag") names.append("file2") c.addFile(names[1], FromString(data[1]), tags[1]) data.append("") tags.append("empty") names.append("") c.addFile(names[2], FromString(data[2]), tags[2]) c.close() c = FileContainer(f) checkFiles(c, names, data, tags) f = util.ExtendedFile(self.fn, "r+", buffering=False) c = FileContainer(f) checkFiles(c, names, data, tags) c.reset() checkFiles(c, names, data, tags) c.close() f = util.ExtendedFile(self.fn, "r+", buffering=False) c = FileContainer(f) name, tag, f = c.getNextFile() assert (name == names[0])
def testLargeFiles(self): # test adding files > 4gig to a filecontainer. we replace the write # call with one which handles sparsity class SparseFile(util.ExtendedFile): def __init__(self, *args, **kwargs): self.needsWrite = False util.ExtendedFile.__init__(self, *args, **kwargs) def write(self, s): if len(s) > 100 and s[0] == '\0' and s[-1] == '\0': self.seek(len(s) - 1, 2) self.needsWrite = True return len(s) return util.ExtendedFile.write(self, s) def close(self): if self.needsWrite: self.write('\0') self.needsWrite = False def seek(self, *args): if self.needsWrite: self.write('\0') self.needsWrite = False return util.ExtendedFile.seek(self, *args) class FalseFile: def __init__(self, size): self.size = size self.offset = 0 def seek(self, offset, whence=0): assert (whence == 0) self.offset = offset def read(self, bytes): self.offset += bytes if self.offset > self.size: self.offset -= bytes bytes = self.size - self.offset self.offset = self.size return "\0" * bytes f = SparseFile(self.fn, "w+", buffering=False) c = FileContainer(f) totalSize = 0x100001000 c.addFile('test', FromFile(FalseFile(totalSize)), 'testdata', precompressed=True) c.addFile('end', FromString('endcontents'), 'enddata', precompressed=True) c.close() c = FileContainer(util.ExtendedFile(self.fn, 'r', buffering=False)) name, tag, f = c.getNextFile() storedSize = f.seek(0, 2) assert (storedSize == totalSize) assert (tag == 'testdata') name, tag, f = c.getNextFile() assert (name == 'end') assert (tag == 'enddata') s = f.read() assert (s == 'endcontents')
def verifySignatures(f, validKeys=None): """ Given an extended file, compute signatures """ f.seek(0) h = readHeader(f) # Cheap test first: verify MD5 sig sigmd5 = h.get(SIG_MD5, None) if sigmd5 is not None: f.seek(0) readSignatureHeader(f) # verify md5 digest md5 = digestlib.md5() util.copyfileobj(f, NullWriter(), digest=md5) if md5.digest() != sigmd5: raise MD5SignatureError( "The MD5 digest fails to verify: expected %s, got %s" % (sha1helper.md5ToString(sigmd5), md5.hexdigest())) # Don't bother if no gpg signature was present, or no valid keys were # presented if validKeys is None: return sigString = h.get(SIG_GPG, None) if sigString is None: return # Skip to immutable header region f.seek(0) readSignatureHeader(f) sig = openpgpfile.readSignature(sigString) keyId = sig.getSignerKeyId() matchingKeys = [x for x in validKeys if x.hasKeyId(keyId)] if not matchingKeys: raise PGPSignatureError("Signature generated with key %s does " "not match valid keys %s" % (keyId, ', '.join(x.getKeyId() for x in validKeys))) key = matchingKeys[0] # signature verification assumes a seekable stream and will seek to the # beginning; use a SeekableNestedFile size = h.getHeaderPlusPayloadSize() if size is None: pos = f.tell() f.seek(0, 2) size = f.tell() f.seek(pos, 0) snf = None if hasattr(f, 'pread'): extFile = f elif hasattr(f, 'name'): extFile = util.ExtendedFile(f.name, buffering=False) else: # worst case scenario, we slurp everything in memory extFile = util.ExtendedStringIO(f.read()) snf = extFile if snf is None: snf = util.SeekableNestedFile(extFile, start=f.tell(), size=size) try: sig.verifyDocument(key.getCryptoKey(), snf) except openpgpfile.SignatureError: raise PGPSignatureError
def get(port, isSecure, repos, req, restHandler=None, authToken=None, repServer=None): uri = req.uri if uri.endswith('/'): uri = uri[:-1] cmd = os.path.basename(uri) if authToken is None: authToken = getAuth(req) if authToken is None: return apache.HTTP_BAD_REQUEST if authToken[0] != "anonymous" and not isSecure and repos.cfg.forceSSL: return apache.HTTP_FORBIDDEN if restHandler and uri.startswith(restHandler.prefix): return restHandler.handle(req, req.unparsed_uri) elif cmd == "changeset": if not req.args: # the client asked for a changeset, but there is no # ?tmpXXXXXX.cf after /conary/changeset (CNY-1142) return apache.HTTP_BAD_REQUEST if '/' in req.args: return apache.HTTP_FORBIDDEN localName = repos.tmpPath + "/" + req.args + "-out" if localName.endswith(".cf-out"): try: f = open(localName, "r") except IOError: return apache.HTTP_NOT_FOUND os.unlink(localName) items = [] totalSize = 0 for l in f.readlines(): (path, size, isChangeset, preserveFile) = l.split() size = int(size) isChangeset = int(isChangeset) preserveFile = int(preserveFile) totalSize += size items.append((path, size, isChangeset, preserveFile)) f.close() del f else: try: size = os.stat(localName).st_size except OSError: return apache.HTTP_NOT_FOUND items = [(localName, size, 0, 0)] totalSize = size # TODO: refactor to use proxy.ChangesetFileReader readNestedFile = proxy.ChangesetFileReader.readNestedFile req.content_type = "application/x-conary-change-set" req.set_content_length(totalSize) for (path, size, isChangeset, preserveFile) in items: if isChangeset: cs = FileContainer(util.ExtendedFile(path, buffering=False)) try: for data in cs.dumpIter(readNestedFile, args=(repos.getContentsStore(), )): req.write(data) except IOError, err: log.error("IOError dumping changeset: %s" % err) return apache.HTTP_BAD_REQUEST del cs else: sendfile(req, size, path) if not preserveFile: os.unlink(path) return apache.OK
csPath = self.workDir + '/test.ccs' #repos.createChangeSetFile( [ #( 'test:runtime', ( old.getVersion(), old.getFlavor() ), #( new.getVersion(), new.getFlavor() ), False ), #], csPath) #fc = filecontainer.FileContainer( #util.ExtendedFile(csPath, buffering = False)) #fc.getNextFile() #assert(not fc.getNextFile()) repos.createChangeSetFile( [ ( 'test:runtime', ( old.getVersion(), old.getFlavor() ), ( new.getVersion(), new.getFlavor() ), False ), ], csPath, mirrorMode = True) fc = filecontainer.FileContainer( util.ExtendedFile(csPath, buffering = False)) fc.getNextFile() if not singleRepos: assert(fc.getNextFile()) repos.createChangeSetFile( [ ( 'test:runtime', ( orig.getVersion(), orig.getFlavor() ), ( new.getVersion(), new.getFlavor() ), False ), ], csPath, mirrorMode = True) cs = changeset.ChangeSetFromFile(csPath) assert( [ x[0] != '\x01' for x in cs.files.values() ] == [ True, True ] ) def testMirrorModeChangesets1(self): self._testMirrorModeChangesets(singleRepos = True)
def open(fn, mode='r', buffering=False): return util.ExtendedFile(fn, mode, buffering)