def addFile(self, fileName, contents, tableData, precompressed = False): assert(isinstance(contents, filecontents.FileContents)) assert(self.mutable) fileObj = contents.get() headerOffset = self.file.tell() self.file.write(struct.pack("!HH", SUBFILE_MAGIC, len(fileName))) self.file.write(struct.pack("!IH", 0, len(tableData))) self.file.write(fileName) self.file.write(tableData) if precompressed: size = util.copyfileobj(fileObj, self.file) else: start = self.file.tell() gzFile = util.DeterministicGzipFile('', "wb", 6, self.file) util.copyfileobj(fileObj, gzFile) gzFile.close() size = self.file.tell() - start if size < 0x100000000: self.file.seek(headerOffset + 4, SEEK_SET) self.file.write(struct.pack("!I", size)) self.file.seek(0, SEEK_END) else: self.file.seek(headerOffset, SEEK_SET) totalSize = size + len(fileName) + len(tableData) self.file.write(struct.pack("!HII", LARGE_SUBFILE_MAGIC, totalSize >> 32, totalSize & 0xFFFFFFFF)) self.file.seek(0, SEEK_END) self.file.write(struct.pack("!HH", len(fileName), len(tableData)))
def _readlineify(self): if hasattr(self.fp, 'readline'): return fp = util.BoundedStringIO() util.copyfileobj(self.fp, fp) fp.seek(0) self.fp = fp
def migrateOneDatabase(self, database): sourceLog = self._openLog('dump') targetLog = self._openLog('restore') source = subprocess.Popen([ self.cluster2.bin('pg_dump'), '--username=postgres', '--port=' + str(self.cluster1.port), '--format=custom', '--compress=0', '--verbose', database, ], shell=False, stdout=subprocess.PIPE, stderr=sourceLog) target = subprocess.Popen([ self.cluster2.bin('pg_restore'), '--username=postgres', '--dbname=' + database, '--port=65001', '--single-transaction', '--verbose', ], shell=False, stdin=subprocess.PIPE, stdout=targetLog, stderr=targetLog) cny_util.copyfileobj(source.stdout, target.stdin) sourceLog.close() targetLog.close() if source.wait(): raise RuntimeError("pg_dumpall failed with exit code %s" % source.returncode) target.stdin.close() if target.wait(): raise RuntimeError("psql failed with exit code %s" % target.returncode)
def _getFile(outPath, inUrl, method='GET', session=None, callback=None): headers = {} if session: headers['Cookie'] = 'vmware_soap_session=%s; $Path=/' % session response = _makeConnection(inUrl, method, headers, callback=callback) if response and response.status not in (200, 201): raise RuntimeError('%s failed: %d - %s' % ( method, response.status, response.reason)) elif not response: raise RuntimeError('%s failed' % method) progress = lambda x, y: x if callback: # Default to the dummy progress callback progress = getattr(callback, 'progress', progress) fileObj = file(outPath, "w") contentLength = response.msg.get('Content-Length') if contentLength is not None: contentLength = int(contentLength) # Chunked transfers will not set Content-Length # we let copyfileobj read to the end util.copyfileobj(response, fileObj, bufSize=BUFSIZE, sizeLimit=contentLength, callback=progress) response.close()
def _sendData(self, conn): if self.data is None: return if not hasattr(self.data, 'read'): conn.send(self.data) return if self.chunked: # Use chunked coding output = wrapper = ChunkedSender(conn) elif self.size is not None: # Use identity coding output = conn wrapper = None else: raise RuntimeError("Request must use chunked transfer coding " "if size is not known.") util.copyfileobj(self.data, output, callback=self.callback, rateLimit=self.rateLimit, abortCheck=self.abortCheck, sizeLimit=self.size) if wrapper: wrapper.close()
def _getFile(outPath, inUrl, method='GET', session=None, callback=None): headers = {} if session: headers['Cookie'] = 'vmware_soap_session=%s; $Path=/' % session response = _makeConnection(inUrl, method, headers, callback=callback) if response and response.status not in (200, 201): raise RuntimeError('%s failed: %d - %s' % (method, response.status, response.reason)) elif not response: raise RuntimeError('%s failed' % method) progress = lambda x, y: x if callback: # Default to the dummy progress callback progress = getattr(callback, 'progress', progress) fileObj = file(outPath, "w") contentLength = response.msg.get('Content-Length') if contentLength is not None: contentLength = int(contentLength) # Chunked transfers will not set Content-Length # we let copyfileobj read to the end util.copyfileobj(response, fileObj, bufSize=BUFSIZE, sizeLimit=contentLength, callback=progress) response.close()
def migrateGlobals(self): log.info("Migrating global data") sourceLog = self._openLog('dump') targetLog = self._openLog('restore') source = subprocess.Popen([ self.cluster2.bin('pg_dumpall'), '--username=postgres', '--port=' + str(self.cluster1.port), '--globals-only', '--no-tablespaces', ], shell=False, stdout=subprocess.PIPE, stderr=sourceLog) target = subprocess.Popen([ self.cluster2.bin('psql'), '--username=postgres', '--dbname=postgres', '--port=65001', '--no-psqlrc', ], shell=False, stdin=subprocess.PIPE, stdout=targetLog, stderr=targetLog) cny_util.copyfileobj(source.stdout, target.stdin) sourceLog.close() targetLog.close() if source.wait(): raise RuntimeError("pg_dumpall failed with exit code %s" % source.returncode) target.stdin.close() if target.wait(): raise RuntimeError("psql failed with exit code %s" % target.returncode)
def _writeFile(cls, fileObj, outFds, precompressed, computeSha1): if precompressed and hasattr(fileObj, '_fdInfo'): (fd, start, size) = fileObj._fdInfo() pid = os.getpid() realHash = digest_uncompress.sha1Copy((fd, start, size), outFds) for x in outFds: cls._fchmod(x) os.close(x) return realHash else: for fd in outFds: outFileObj = os.fdopen(fd, "w") contentSha1 = digestlib.sha1() if precompressed and computeSha1: tee = Tee(fileObj, outFileObj) uncompObj = gzip.GzipFile(mode = "r", fileobj = tee) s = uncompObj.read(128 * 1024) while s: contentSha1.update(s) s = uncompObj.read(128 * 1024) uncompObj.close() elif precompressed: util.copyfileobj(fileObj, outFileObj) else: dest = gzip.GzipFile(mode = "w", fileobj = outFileObj) util.copyfileobj(fileObj, dest, digest = contentSha1) dest.close() # this closes tmpFd for us cls._fchmod(fd) outFileObj.close() fileObj.seek(0) return contentSha1.digest()
def _readlineify(self): if hasattr(self.fp, 'readline'): return fp = util.BoundedStringIO() util.copyfileobj(self.fp, fp) fp.seek(0) self.fp = fp
def restore(self, fileContents, root, target, journal=None, sha1=None, nameLookup=True, **kwargs): keepTempfile = kwargs.get('keepTempfile', False) destTarget = target if fileContents is not None: # this is first to let us copy the contents of a file # onto itself; the unlink helps that to work src = fileContents.get() inFd = None if fileContents.isCompressed() and hasattr(src, '_fdInfo'): # inFd is None if we can't figure this information out # (for _LazyFile for instance) (inFd, inStart, inSize) = src._fdInfo() path, name = os.path.split(target) if not os.path.isdir(path): util.mkdirChain(path) # Uncompress to a temporary file, using the accelerated # implementation if possible. if inFd is not None and util.sha1Uncompress is not None: actualSha1, tmpname = util.sha1Uncompress( inFd, inStart, inSize, path, name) else: if fileContents.isCompressed(): src = gzip.GzipFile(mode='r', fileobj=src) tmpfd, tmpname = tempfile.mkstemp(name, '.ct', path) try: d = digestlib.sha1() f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f, digest=d) f.close() actualSha1 = d.digest() except: os.unlink(tmpname) raise if keepTempfile: # Make a hardlink "copy" for the caller to use destTarget = tmpname + '.ptr' os.link(tmpname, destTarget) try: os.rename(tmpname, target) except OSError, err: if err.args[0] != errno.EISDIR: raise os.rmdir(target) os.rename(tmpname, target) if (sha1 is not None and sha1 != actualSha1): raise Sha1Exception(target)
def _writeFile(cls, fileObj, outFds, precompressed, computeSha1): if precompressed and hasattr(fileObj, '_fdInfo'): (fd, start, size) = fileObj._fdInfo() pid = os.getpid() realHash = digest_uncompress.sha1Copy((fd, start, size), outFds) for x in outFds: cls._fchmod(x) os.close(x) return realHash else: for fd in outFds: outFileObj = os.fdopen(fd, "w") contentSha1 = digestlib.sha1() if precompressed and computeSha1: tee = Tee(fileObj, outFileObj) uncompObj = gzip.GzipFile(mode="r", fileobj=tee) s = uncompObj.read(128 * 1024) while s: contentSha1.update(s) s = uncompObj.read(128 * 1024) uncompObj.close() elif precompressed: util.copyfileobj(fileObj, outFileObj) else: dest = gzip.GzipFile(mode="w", fileobj=outFileObj) util.copyfileobj(fileObj, dest, digest=contentSha1) dest.close() # this closes tmpFd for us cls._fchmod(fd) outFileObj.close() fileObj.seek(0) return contentSha1.digest()
def request(self, method, body=None, headers=None, contentLength=None, callback=None): hdrs = self.headers.copy() hdrs.update(headers or {}) if self.user is not None and self.passwd is not None: user_pass = base64.b64encode('%s:%s' % (self.user, self.passwd)) hdrs['Authorization'] = 'Basic %s' % user_pass if hasattr(body, "read"): # We need to stream if contentLength is None: # Determine body size body.seek(0, 2) contentLength = body.tell() body.seek(0, 0) hdrs['Content-Length'] = str(contentLength) self._connection.putrequest(method, self.path) for hdr, value in hdrs.iteritems(): self._connection.putheader(hdr, value) self._connection.endheaders() util.copyfileobj(body, self._connection, sizeLimit=contentLength, callback=callback) else: self._connection.request(method, self.path, body = body, headers = hdrs) resp = self._connection.getresponse() if resp.status != 200: raise ResponseError(resp.status, resp.reason, resp.msg, resp) return resp
def apply(self, justDatabase=False, noScripts=False, capsuleChangeSet=None): if capsuleChangeSet: # Previous jobs will have moved the pointer in the auxilliary # changeset, so reset it at the start of each job. capsuleChangeSet.reset() tmpDir = os.path.join(self.root, 'var/tmp') if not os.path.isdir(tmpDir): # For empty roots or roots that are not systems (e.g. source # checkouts), just put capsules in the root directory. tmpDir = self.root fileDict = {} for kind, obj in sorted(self.capsuleClasses.items()): fileDict.update( dict(((x[0], x[2], x[3]), x[1]) for x in obj._filesNeeded())) try: for ((pathId, fileId, sha1), path) in sorted(fileDict.items()): tmpfd, tmpname = tempfile.mkstemp(dir=tmpDir, prefix=path, suffix='.conary') fType, fContents = self.changeSet.getFileContents( pathId, fileId) if (fType == changeset.ChangedFileTypes.hldr): if (capsuleChangeSet): try: result = capsuleChangeSet.getFileContents( pathId, fileId) fObj = result[1].get() except KeyError: raise errors.MissingRollbackCapsule( 'Cannot find ' 'RPM %s to perform local rollback' % path) else: fObj = fContents.get() d = digestlib.sha1() util.copyfileobj(fObj, os.fdopen(tmpfd, "w"), digest=d) actualSha1 = d.digest() if actualSha1 != sha1: raise files.Sha1Exception(path) # tmpfd is closed when the file object created by os.fdopen # disappears fileDict[(pathId, fileId)] = tmpname for kind, obj in sorted(self.capsuleClasses.items()): obj.apply(fileDict, justDatabase=justDatabase, noScripts=noScripts) finally: for tmpPath in fileDict.values(): try: os.unlink(tmpPath) except: pass
def store(cls, root, archive): """ Copy a manifest from inside a chroot to adjacent to a file archive """ f_in = open(os.path.join(root, cls.FILENAME)) with util.AtomicFile(archive + cls.AR_SUFFIX) as f_out: util.copyfileobj(f_in, f_out) f_in.close()
def store(cls, root, archive): """ Copy a manifest from inside a chroot to adjacent to a file archive """ f_in = open(os.path.join(root, cls.FILENAME)) with util.AtomicFile(archive + cls.AR_SUFFIX) as f_out: util.copyfileobj(f_in, f_out) f_in.close()
def fetchArchive(self, conarySource, snapPath): if os.path.exists(snapPath): return archive = urllib.quote(os.path.basename(snapPath)) log.info("Downloading %s", archive) f_in = self.wms.archive(self.path, self.revision, archive) with open(snapPath, 'w') as f_out: copyfileobj(f_in, f_out) f_in.close()
def fetchArchive(self, conarySource, snapPath): if os.path.exists(snapPath): return archive = urllib.quote(os.path.basename(snapPath)) log.info("Downloading %s", archive) f_in = self.wms.archive(self.path, self.revision, archive) with open(snapPath, 'w') as f_out: copyfileobj(f_in, f_out) f_in.close()
def fetchArchive(self, conarySource, snapPath): if os.path.exists(snapPath): return archive = urllib.quote(os.path.basename(snapPath)) url = (self.repos + '/archive/' + urllib.quote(self.revision) + '/' + archive) f_in = self.opener.open(url) with open(snapPath, 'w') as f_out: copyfileobj(f_in, f_out) f_in.close()
def restore(self, fileContents, root, target, journal=None, sha1 = None, nameLookup=True, **kwargs): keepTempfile = kwargs.get('keepTempfile', False) destTarget = target if fileContents is not None: # this is first to let us copy the contents of a file # onto itself; the unlink helps that to work src = fileContents.get() inFd = None if fileContents.isCompressed() and hasattr(src, '_fdInfo'): # inFd is None if we can't figure this information out # (for _LazyFile for instance) (inFd, inStart, inSize) = src._fdInfo() path, name = os.path.split(target) if not os.path.isdir(path): util.mkdirChain(path) # Uncompress to a temporary file, using the accelerated # implementation if possible. if inFd is not None and util.sha1Uncompress is not None: actualSha1, tmpname = util.sha1Uncompress( inFd, inStart, inSize, path, name) else: if fileContents.isCompressed(): src = gzip.GzipFile(mode='r', fileobj=src) tmpfd, tmpname = tempfile.mkstemp(name, '.ct', path) try: d = digestlib.sha1() f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f, digest = d) f.close() actualSha1 = d.digest() except: os.unlink(tmpname) raise if keepTempfile: # Make a hardlink "copy" for the caller to use destTarget = tmpname + '.ptr' os.link(tmpname, destTarget) try: os.rename(tmpname, target) except OSError, err: if err.args[0] != errno.EISDIR: raise os.rmdir(target) os.rename(tmpname, target) if (sha1 is not None and sha1 != actualSha1): raise Sha1Exception(target)
def __init__(self, path): self.path = path self.f_plain = open(path, 'a', buffering=0) if os.path.exists(path + '.gz') and not os.fstat( self.f_plain.fileno()).st_size: # The compressed log exists already, but the plain log was removed. # Need to copy the compressed contents to the plain log. f_gz = gzip.GzipFile(path + '.gz', 'r') util.copyfileobj(f_gz, self.f_plain) f_gz.close() self.f_gz = gzip.GzipFile(path + '.gz', 'a')
def _makeConnection(url, method, headers=None, bodyStream=None, bodyLength=None, callback=None): protocol, uri = urllib.splittype(url) assert (protocol in ('http', 'https')) host, selector = urllib.splithost(uri) host, port = urllib.splitport(host) if protocol == 'http': r = HTTPConnection(host, port) else: r = HTTPSConnection(host, port) progress = None if callback: progress = getattr(callback, 'progress', None) if progress is None: progress = lambda x, y: x hdrs = {'Content-Type': 'application/octet-stream'} hdrs.update(headers or {}) if bodyStream and bodyLength is None: bodyStream.seek(0, 2) bodyLength = bodyStream.tell() bodyStream.seek(0) if bodyLength: hdrs['Content-Length'] = bodyLength #r.set_debuglevel(1) r.connect() r.putrequest(method, selector) for k, v in hdrs.items(): r.putheader(k, str(v)) try: r.endheaders() if bodyStream: # This could fail, if the device backed by this file is connected util.copyfileobj(bodyStream, r, bufSize=BUFSIZE, sizeLimit=hdrs['Content-Length'], callback=progress) return r.getresponse() except socket.error, e: raise response = None try: response = r.getresponse() r.close() except Exception, e: pass
def __init__(self, path): self.path = path self.f_plain = open(path, 'a', buffering=0) if os.path.exists(path + '.gz' ) and not os.fstat(self.f_plain.fileno()).st_size: # The compressed log exists already, but the plain log was removed. # Need to copy the compressed contents to the plain log. f_gz = gzip.GzipFile(path + '.gz', 'r') util.copyfileobj(f_gz, self.f_plain) f_gz.close() self.f_gz = gzip.GzipFile(path + '.gz', 'a')
def compressDiskImage(self, vmdkPath): if not self.WithCompressedDisks: # Need to add the file to the final directory destf = util.AtomicFile(os.path.join(self.outputDir, self.basefilename + '.vmdk')) util.copyfileobj(file(vmdkPath), destf) destf.commit() return destf.finalPath vmdkGzOutputFile = os.path.join(self.outputDir, self.basefilename + '.vmdk.gz') self.gzip(vmdkPath, vmdkGzOutputFile) util.remove(vmdkPath) return vmdkGzOutputFile
def apply(self, justDatabase = False, noScripts = False, capsuleChangeSet = None): if capsuleChangeSet: # Previous jobs will have moved the pointer in the auxilliary # changeset, so reset it at the start of each job. capsuleChangeSet.reset() tmpDir = os.path.join(self.root, 'var/tmp') if not os.path.isdir(tmpDir): # For empty roots or roots that are not systems (e.g. source # checkouts), just put capsules in the root directory. tmpDir = self.root fileDict = {} for kind, obj in sorted(self.capsuleClasses.items()): fileDict.update( dict(((x[0], x[2], x[3]), x[1]) for x in obj._filesNeeded())) try: for ((pathId, fileId, sha1), path) in sorted(fileDict.items()): tmpfd, tmpname = tempfile.mkstemp(dir=tmpDir, prefix=path, suffix='.conary') fType, fContents = self.changeSet.getFileContents(pathId, fileId) if (fType == changeset.ChangedFileTypes.hldr): if (capsuleChangeSet): try: result = capsuleChangeSet.getFileContents(pathId, fileId) fObj = result[1].get() except KeyError: raise errors.MissingRollbackCapsule('Cannot find ' 'RPM %s to perform local rollback' % path) else: fObj = fContents.get() d = digestlib.sha1() util.copyfileobj(fObj, os.fdopen(tmpfd, "w"), digest = d) actualSha1 = d.digest() if actualSha1 != sha1: raise files.Sha1Exception(path) # tmpfd is closed when the file object created by os.fdopen # disappears fileDict[(pathId, fileId)] = tmpname for kind, obj in sorted(self.capsuleClasses.items()): obj.apply(fileDict, justDatabase = justDatabase, noScripts = noScripts) finally: for tmpPath in fileDict.values(): try: os.unlink(tmpPath) except: pass
def __init__(self, sourceTrove, repos, cfg, versionStr=None, labelPath=None, ignoreInstalled=False, filterVersions=False, parentDir=None, defaultToLatest = False, buildFlavor = None, db = None, overrides = None, getFileFunction = None, branch = None): self.recipes = {} if getFileFunction is None: getFileFunction = lambda repos, fileId, fileVersion, path: \ repos.getFileContents([ (fileId, fileVersion) ])[0].get() name = sourceTrove.getName().split(':')[0] recipePath = name + '.recipe' match = self.findFileByPath(sourceTrove, recipePath) if not match: # this is just missing the recipe; we need it raise builderrors.RecipeFileError("version %s of %s does not " "contain %s" % (sourceTrove.getName(), sourceTrove.getVersion().asString(), recipePath)) (fd, recipeFile) = tempfile.mkstemp(".recipe", 'temp-%s-' %name, dir=cfg.tmpDir) outF = os.fdopen(fd, "w") inF = getFileFunction(repos, match[0], match[1], recipePath) util.copyfileobj(inF, outF) del inF outF.close() del outF if branch is None: branch = sourceTrove.getVersion().branch() try: loadrecipe.RecipeLoader.__init__(self, recipeFile, cfg, repos, sourceTrove.getName(), branch = branch, ignoreInstalled=ignoreInstalled, directory=parentDir, buildFlavor=buildFlavor, db=db, overrides=overrides) finally: os.unlink(recipeFile) self.recipe._trove = sourceTrove.copy()
def _downloadParentImage(self, imgSpec, unpackDir, layersDir): log.debug('Downloading parent image %s', imgSpec.dockerImageId) self.status('Downloading parent image') resp = self.response.getImage(imgSpec.url) tmpf = tempfile.TemporaryFile(dir=self.workDir) util.copyfileobj(resp, tmpf) tmpf.seek(0) self.status('Unpacking parent image') errcode, stdout, stderr = logCall(["tar", "-C", layersDir, "-zxf", "-"], stdin=tmpf) tmpf.close() parentImageDir = os.path.join(unpackDir, imgSpec.dockerImageId) log.debug('Unpacking parent image as %s', self._path(parentImageDir)) layerFilesStack = [] # Unpack the layers in some temporary directories layer = imgSpec while layer is not None: layerId = layer.dockerImageId layer._unpackDir = parentImageDir layerFilesStack.append( (layerId, os.path.join(layersDir, layerId, "layer.tar"))) layer.layerSize = os.stat(layerFilesStack[-1][1]).st_size layer._manifest = mf = Manifest(json.load(file(os.path.join(layersDir, layerId, 'json')))) mf = json.load(file(os.path.join(layersDir, layerId, 'json'))) parent = mf.get('parent') if parent is not None and not layer.parent: layer.parent = ImageSpec(dockerImageId=parent) layer.parent.children.append(layer) layer = layer.parent # We now extract all layers, top-to-bottom, in the same directory. while layerFilesStack: layerId, layerFile = layerFilesStack.pop() log.debug(" Extracting parent layer %s on %s", layerId, self._path(parentImageDir)) self._extractLayer(parentImageDir, layerFile) idToNameTags = {} reposFile = os.path.join(layersDir, 'repositories') if os.path.isfile(reposFile): repos = json.load(file(reposFile)) for name, tagToId in repos.iteritems(): for tag, imgid in tagToId.iteritems(): idToNameTags.setdefault(imgid, set()).add((name, tag)) # Walk list again, to compute tags layer = imgSpec while layer is not None: layerId = layer.dockerImageId layer.updateNamesAndTags(idToNameTags.get(layerId)) layer = layer.parent
def _generate(self): self._log.info("Generating template %s from trove %s=%s[%s]", self._hash, *self._troveTup) self._installContents(self._contentsDir, [self._troveTup]) # Copy "unified" directly into the output. os.mkdir(self._outputDir) util.copytree(self._contentsDir + '/unified', self._outputDir + '/') # Process the MANIFEST file. for line in open(self._contentsDir + '/MANIFEST'): line = line.rstrip() if not line or line[0] == '#': continue args = line.rstrip().split(',') command = args.pop(0) commandFunc = getattr(self, '_DO_' + command, None) if not commandFunc: raise RuntimeError("Unknown command %r in MANIFEST" % (command,)) commandFunc(args) # Archive the results. digest = digestlib.sha1() outFile = util.AtomicFile(self._outputPath) proc = call(['/bin/tar', '-cC', self._outputDir, '.'], stdout=subprocess.PIPE, captureOutput=False, wait=False) util.copyfileobj(proc.stdout, outFile, digest=digest) proc.wait() # Write metadata. metaFile = util.AtomicFile(self._outputPath + '.metadata') cPickle.dump({ 'sha1sum': digest.hexdigest(), 'trovespec': '%s=%s[%s]' % self._troveTup, 'kernel': (self._kernelTup and ('%s=%s[%s]' % self._kernelTup) or '<none>'), # Right now, we are going to hardcode this to an older version # of Netclient Protocol to hint to the Conary installed on the # jobslave to generate old filecontainers that are compatible # with all versions of Conary. (See RBL-1552.) 'netclient_protocol_version': '38', }, metaFile) metaFile.commit() outFile.commit() self._log.info("Template %s created", self._hash)
def do_PUT(self): chunked = False if 'Transfer-encoding' in self.headers: contentLength = 0 chunked = True elif 'Content-Length' in self.headers: chunked = False contentLength = int(self.headers['Content-Length']) else: # send 411: Length Required self.send_error(411) authToken = self.getAuth() if self.cfg.proxyContentsDir: status, reason = netclient.httpPutFile(self.path, self.rfile, contentLength) self.send_response(status) return path = self.path.split("?")[-1] if '/' in path: self.send_error(403) path = self.tmpDir + '/' + path + "-in" size = os.stat(path).st_size if size != 0: self.send_error(410) return out = open(path, "w") try: if chunked: while 1: chunk = self.rfile.readline() chunkSize = int(chunk, 16) # chunksize of 0 means we're done if chunkSize == 0: break util.copyfileobj(self.rfile, out, sizeLimit=chunkSize) # read the \r\n after the chunk we just copied self.rfile.readline() else: util.copyfileobj(self.rfile, out, sizeLimit=contentLength) finally: out.close() self.send_response(200) self.end_headers()
def do_PUT(self): chunked = False if 'Transfer-encoding' in self.headers: contentLength = 0 chunked = True elif 'Content-Length' in self.headers: chunked = False contentLength = int(self.headers['Content-Length']) else: # send 411: Length Required self.send_error(411) authToken = self.getAuth() if self.cfg.proxyContentsDir: status, reason = netclient.httpPutFile(self.path, self.rfile, contentLength) self.send_response(status) return path = self.path.split("?")[-1] if '/' in path: self.send_error(403) path = self.tmpDir + '/' + path + "-in" size = os.stat(path).st_size if size != 0: self.send_error(410) return out = open(path, "w") try: if chunked: while 1: chunk = self.rfile.readline() chunkSize = int(chunk, 16) # chunksize of 0 means we're done if chunkSize == 0: break util.copyfileobj(self.rfile, out, sizeLimit=chunkSize) # read the \r\n after the chunk we just copied self.rfile.readline() else: util.copyfileobj(self.rfile, out, sizeLimit=contentLength) finally: out.close() self.send_response(200) self.end_headers()
def testCorruptedRpm(self): # test enforcement of size and sha1 fields from rpm signature block f = open(resources.get_archive() + "/basesystem-8.0-2.src.rpm") rpmhelper.readHeader(f) f.seek(0) # change the size tmp = tempfile.TemporaryFile() util.copyfileobj(f, tmp) f.seek(0) tmp.write(" ") tmp.seek(0) try: rpmhelper.readHeader(tmp) except IOError, e: assert str(e) == "file size does not match size specified by " "header"
def _makeConnection(url, method, headers = None, bodyStream = None, bodyLength = None, callback = None): protocol, uri = urllib.splittype(url) assert(protocol in ('http', 'https')) host, selector = urllib.splithost(uri) host, port = urllib.splitport(host) if protocol == 'http': r = HTTPConnection(host, port) else: r = HTTPSConnection(host, port) progress = None if callback: progress = getattr(callback, 'progress', None) if progress is None: progress = lambda x, y: x hdrs = { 'Content-Type' : 'application/octet-stream'} hdrs.update(headers or {}) if bodyStream and bodyLength is None: bodyStream.seek(0, 2) bodyLength = bodyStream.tell() bodyStream.seek(0) if bodyLength: hdrs['Content-Length'] = bodyLength #r.set_debuglevel(1) r.connect() r.putrequest(method, selector) for k, v in hdrs.items(): r.putheader(k, str(v)) try: r.endheaders() if bodyStream: # This could fail, if the device backed by this file is connected util.copyfileobj(bodyStream, r, bufSize=BUFSIZE, sizeLimit=hdrs['Content-Length'], callback=progress) return r.getresponse() except socket.error, e: raise response = None try: response = r.getresponse() r.close() except Exception, e: pass
def testCorruptedRpm(self): # test enforcement of size and sha1 fields from rpm signature block f = open(resources.get_archive() + "/basesystem-8.0-2.src.rpm") rpmhelper.readHeader(f) f.seek(0) # change the size tmp = tempfile.TemporaryFile() util.copyfileobj(f, tmp) f.seek(0) tmp.write(' ') tmp.seek(0) try: rpmhelper.readHeader(tmp) except IOError, e: assert (str(e) == 'file size does not match size specified by ' 'header')
def _getFilesystemImage(self, job, image, stream): dlfile = tempfile.NamedTemporaryFile(prefix=image.getBaseFileName(), delete=False) util.copyfileobj(stream, dlfile) dlfile.close() dlpath = dlfile.name fileExtensions = [ '.ext3' ] self._msg(job, "Uncompressing image") workdir = self.extractImage(dlpath) # XXX make this more robust imageFileDir, imageFileName = self.findFile(workdir, fileExtensions) if imageFileDir is None: raise RuntimeError("No file(s) found: %s" % ', '.join("*%s" % x for x in fileExtensions)) imageFilePath = os.path.join(imageFileDir, imageFileName) return imageFilePath
def write(self): # Output setup topDir = os.path.join(self.workDir, 'ova_base') util.mkdirChain(topDir) outputDir = os.path.join(constants.finishedDir, self.UUID) util.mkdirChain(outputDir) deliverable = os.path.join(outputDir, self.basefilename + self.suffix) # Build the filesystem images image_path = os.path.join(self.workDir, 'hdimage') disk = self.makeHDImage(image_path) # Open a manifest for tar so that it writes out files in the optimal # order. manifest_path = os.path.join(self.workDir, 'files') manifest = open(manifest_path, 'w') # Write the ova.xml file ovaName = 'ova.xml' ovaPath = os.path.join(topDir, ovaName) self.createXVA(ovaPath, disk.totalSize) print >>manifest, ovaName # Split the HD image into 1GB (not GiB) chunks label = 'xvda' chunk_dir = os.path.join(topDir, label) chunkPrefix = os.path.join(chunk_dir, 'chunk-') util.mkdirChain(os.path.split(chunkPrefix)[0]) self.status('Splitting hard disk image') infile = open(image_path, 'rb') n = 0 tocopy = os.stat(image_path).st_size while True: chunkname = '%s%04d.gz' % (chunkPrefix, n) outfile = gzip.GzipFile(chunkname, 'wb') tocopy -= util.copyfileobj(infile, outfile, sizeLimit=1000000000) outfile.close() print >>manifest, chunkname if not tocopy: break n += 1 infile.close() # Delete the FS image to free up temp space os.unlink(image_path) # Create XVA file manifest.close() self.status('Creating XVA Image') logCall('tar -cv -f "%s" -C "%s" -T "%s"' % \ (deliverable, topDir, manifest_path)) self.outputFileList.append((deliverable, 'Citrix XenServer (TM) Image'),) self.postOutput(self.outputFileList)
def extractFilesFromCpio(fileIn, fileList, tmpDir = '/tmp'): """ Returns a list of open files parallel to fileList Hardlinked files will share contents, so make sure you seek() back to the beginning before you read. """ # Map device/inode to catch hardlinks inodeMap = {} # Map the path in fileList to header and device/inode fileNameMap = dict((_normpath(x), x) for x in fileList) fileNameInodeMap = {} # Empty files will be shared to avoid consuming fd EmptyFile = tempfile.TemporaryFile(dir = tmpDir, prefix = 'tmppayload-') cpioObj = cpiostream.CpioStream(fileIn) for entry in cpioObj: if entry.header.mode & 0170000 != 0100000: # Not a regular file continue fileName = _normpath(entry.filename) devmajor = entry.header.devmajor devminor = entry.header.devminor inode = entry.header.inode key = (devmajor, devminor, inode) # This file may not be the one we're looking for, but it may be the # one that provides the contents for hardlinked files we care about if fileName not in fileNameMap and key not in inodeMap: continue if entry.header.filesize == 0: fobj = EmptyFile else: fobj = tempfile.TemporaryFile(dir = tmpDir, prefix = 'tmppayload-') util.copyfileobj(entry.payload, fobj) fobj.seek(0) inodeMap[key] = fobj # in case we'll ever want to use the information from the cpio header # entry to restore file permissions, we should also save the header # here fileNameInodeMap[fileName] = key
def extractFilesFromCpio(fileIn, fileList, tmpDir='/tmp'): """ Returns a list of open files parallel to fileList Hardlinked files will share contents, so make sure you seek() back to the beginning before you read. """ # Map device/inode to catch hardlinks inodeMap = {} # Map the path in fileList to header and device/inode fileNameMap = dict((_normpath(x), x) for x in fileList) fileNameInodeMap = {} # Empty files will be shared to avoid consuming fd EmptyFile = tempfile.TemporaryFile(dir=tmpDir, prefix='tmppayload-') cpioObj = cpiostream.CpioStream(fileIn) for entry in cpioObj: if entry.header.mode & 0170000 != 0100000: # Not a regular file continue fileName = _normpath(entry.filename) devmajor = entry.header.devmajor devminor = entry.header.devminor inode = entry.header.inode key = (devmajor, devminor, inode) # This file may not be the one we're looking for, but it may be the # one that provides the contents for hardlinked files we care about if fileName not in fileNameMap and key not in inodeMap: continue if entry.header.filesize == 0: fobj = EmptyFile else: fobj = tempfile.TemporaryFile(dir=tmpDir, prefix='tmppayload-') util.copyfileobj(entry.payload, fobj) fobj.seek(0) inodeMap[key] = fobj # in case we'll ever want to use the information from the cpio header # entry to restore file permissions, we should also save the header # here fileNameInodeMap[fileName] = key
def get(self, pathId): f = self.fileClass(pathId) f.inode = files.InodeStream(self.perms & 07777, self.mtime, self.owner, self.group) self._touchupFileStream(f) if self.needSha1: sha1 = digestlib.sha1() contents = self.contents.get() devnull = open(os.devnull, "w") util.copyfileobj(contents, devnull, digest=sha1) devnull.close() f.contents = files.RegularFileStream() f.contents.size.set(contents.tell()) f.contents.sha1.set(sha1.digest()) f.provides.set(self.provides) f.requires.set(self.requires) f.flavor.set(self.flavor) for tag in self.tags: f.tags.set(tag) return f
def get(self, pathId): f = self.fileClass(pathId) f.inode = files.InodeStream(self.perms & 07777, self.mtime, self.owner, self.group) self._touchupFileStream(f) if self.needSha1: sha1 = digestlib.sha1() contents = self.contents.get() devnull = open(os.devnull, 'w') util.copyfileobj(contents, devnull, digest=sha1) devnull.close() f.contents = files.RegularFileStream() f.contents.size.set(contents.tell()) f.contents.sha1.set(sha1.digest()) f.provides.set(self.provides) f.requires.set(self.requires) f.flavor.set(self.flavor) for tag in self.tags: f.tags.set(tag) return f
def putChangeset(self): """PUT method -- handle changeset uploads.""" if not self.repositoryServer: # FIXME: this mechanism is unauthenticated and can probably be used # to PUT content to random things on the internet if 'content-length' in self.request.headers: size = int(self.request.headers['content-length']) else: size = None headers = [ x for x in self.request.headers.items() if x[0].lower() in ( 'x-conary-servername', 'x-conary-entitlement', ) ] result = netclient.httpPutFile( self.request.url, self.request.body_file, size, headers=headers, chunked=(size is None), withResponse=True, ) return self.responseFactory( status='%s %s' % (result.status, result.reason), app_iter=self._produceProxy(result), #headerlist=result.getheaders(), ) # Copy request body to the designated temporary file. stream = self.request.body_file out = self._openForPut() if out is None: # File already exists or is in an illegal location. return self._makeError('403 Forbidden', "Illegal changeset upload") util.copyfileobj(stream, out) out.close() return self.responseFactory(status='200 OK')
def cacheFilePath(self, cachePrefix, url): cachePath = self.getCachePath(cachePrefix, url) util.mkdirChain(os.path.dirname(cachePath)) if url.filePath() in self.cacheMap: # don't check sha1 twice return self.cacheMap[url.filePath()] (troveName, troveVersion, pathId, troveFile, fileId, troveFileVersion, sha1, mode) = self.nameMap[url.filePath()] sha1Cached = None cachedMode = None if os.path.exists(cachePath): sha1Cached = sha1helper.sha1FileBin(cachePath) if sha1Cached != sha1: if sha1Cached: log.info('%s sha1 %s != %s; fetching new...', url.filePath(), sha1helper.sha1ToString(sha1), sha1helper.sha1ToString(sha1Cached)) else: log.info('%s not yet cached, fetching...', url.filePath()) if self.quiet: csCallback = None else: csCallback = ChangesetCallback() f = self.repos.getFileContents([(fileId, troveFileVersion)], callback=csCallback)[0].get() outF = util.AtomicFile(cachePath, chmod=0644) util.copyfileobj(f, outF) outF.commit() fileObj = self.repos.getFileVersion(pathId, fileId, troveFileVersion) fileObj.chmod(cachePath) cachedMode = os.stat(cachePath).st_mode & 0777 if mode != cachedMode: os.chmod(cachePath, mode) self.cacheMap[url.filePath()] = cachePath return cachePath
def cacheFilePath(self, cachePrefix, url): cachePath = self.getCachePath(cachePrefix, url) util.mkdirChain(os.path.dirname(cachePath)) if url.filePath() in self.cacheMap: # don't check sha1 twice return self.cacheMap[url.filePath()] (troveName, troveVersion, pathId, troveFile, fileId, troveFileVersion, sha1, mode) = self.nameMap[url.filePath()] sha1Cached = None cachedMode = None if os.path.exists(cachePath): sha1Cached = sha1helper.sha1FileBin(cachePath) if sha1Cached != sha1: if sha1Cached: log.info('%s sha1 %s != %s; fetching new...', url.filePath(), sha1helper.sha1ToString(sha1), sha1helper.sha1ToString(sha1Cached)) else: log.info('%s not yet cached, fetching...', url.filePath()) if self.quiet: csCallback = None else: csCallback = ChangesetCallback() f = self.repos.getFileContents( [(fileId, troveFileVersion)], callback=csCallback)[0].get() outF = util.AtomicFile(cachePath, chmod=0644) util.copyfileobj(f, outF) outF.commit() fileObj = self.repos.getFileVersion( pathId, fileId, troveFileVersion) fileObj.chmod(cachePath) cachedMode = os.stat(cachePath).st_mode & 0777 if mode != cachedMode: os.chmod(cachePath, mode) self.cacheMap[url.filePath()] = cachePath return cachePath
def _sendData(self, conn): if self.data is None: return if not hasattr(self.data, 'read'): conn.send(self.data) return if self.chunked: # Use chunked coding output = wrapper = ChunkedSender(conn) elif self.size is not None: # Use identity coding output = conn wrapper = None else: raise RuntimeError("Request must use chunked transfer coding " "if size is not known.") util.copyfileobj(self.data, output, callback=self.callback, rateLimit=self.rateLimit, abortCheck=self.abortCheck, sizeLimit=self.size) if wrapper: wrapper.close()
def addFileToCache(self, cachePrefix, url, infile, contentLength): # cache needs to be hierarchical to avoid collisions, thus we # use cachePrefix so that files with the same name and different # contents in different packages do not collide cachedname = self.getCachePath(cachePrefix, url) util.mkdirChain(os.path.dirname(cachedname)) f = util.AtomicFile(cachedname, chmod=0644) try: BLOCKSIZE = 1024 * 4 if self.quiet: callback = callbacks.FetchCallback() else: callback = FetchCallback() wrapper = callbacks.CallbackRateWrapper(callback, callback.fetch, contentLength) util.copyfileobj(infile, f, bufSize=BLOCKSIZE, rateLimit=self.downloadRateLimit, callback=wrapper.callback) f.commit() infile.close() except: f.close() raise # work around FTP bug (msw had a better way?) if url.scheme == 'ftp': if os.stat(cachedname).st_size == 0: os.unlink(cachedname) self.createNegativeCacheEntry(cachePrefix, url) return None return cachedname
def putChangeset(self): """PUT method -- handle changeset uploads.""" if not self.repositoryServer: # FIXME: this mechanism is unauthenticated and can probably be used # to PUT content to random things on the internet if 'content-length' in self.request.headers: size = int(self.request.headers['content-length']) else: size = None headers = [x for x in self.request.headers.items() if x[0].lower() in ( 'x-conary-servername', 'x-conary-entitlement', )] result = netclient.httpPutFile(self.request.url, self.request.body_file, size, headers=headers, chunked=(size is None), withResponse=True, ) return self.responseFactory( status='%s %s' % (result.status, result.reason), app_iter=self._produceProxy(result), #headerlist=result.getheaders(), ) # Copy request body to the designated temporary file. stream = self.request.body_file out = self._openForPut() if out is None: # File already exists or is in an illegal location. return self._makeError('403 Forbidden', "Illegal changeset upload") util.copyfileobj(stream, out) out.close() return self.responseFactory(status='200 OK')
def get(self, fileName, computeShaDigest = False): """ Download a file from the repository. @param fileName: relative path to file @type fileName: string @return open file instance """ fobj = self._getTempFileObject() realUrl = self._getRealUrl(fileName) inf = self._opener.open(realUrl) if computeShaDigest: dig = digestlib.sha1() else: dig = None util.copyfileobj(inf, fobj, digest = dig) fobj.seek(0) if not os.path.basename(fileName).endswith('.gz'): return self.FileWrapper.create(fobj, dig) return self.FileWrapper.create(gzip.GzipFile(fileobj=fobj, mode="r"), dig)
def testCopyFileObjDigest(self): tmpDir = tempfile.mkdtemp() try: buf = "test data" # rpepare source and destination files srcFn = os.path.join(tmpDir, "srcfile") destFn = os.path.join(tmpDir, "destfile") open(srcFn, "w").write(buf) src = open(srcFn) dest = open(destFn, "w") # filter the digest through copyfileobj sha1 = digestlib.sha1() util.copyfileobj(src, dest, digest=sha1, sizeLimit=len(buf)) res = sha1.hexdigest() # now compare the resulting hash to reference data sha1 = digestlib.sha1() sha1.update(buf) ref = sha1.hexdigest() self.assertEquals(ref, res) finally: util.rmtree(tmpDir)
def addFileToCache(self, cachePrefix, url, infile, contentLength): # cache needs to be hierarchical to avoid collisions, thus we # use cachePrefix so that files with the same name and different # contents in different packages do not collide cachedname = self.getCachePath(cachePrefix, url) util.mkdirChain(os.path.dirname(cachedname)) f = open(cachedname, "w+") try: BLOCKSIZE = 1024 * 4 if self.quiet: callback = callbacks.FetchCallback() else: callback = FetchCallback() wrapper = callbacks.CallbackRateWrapper(callback, callback.fetch, contentLength) util.copyfileobj(infile, f, bufSize=BLOCKSIZE, rateLimit=self.downloadRateLimit, callback=wrapper.callback) f.close() infile.close() except: os.unlink(cachedname) raise # work around FTP bug (msw had a better way?) if url.scheme == 'ftp': if os.stat(cachedname).st_size == 0: os.unlink(cachedname) self.createNegativeCacheEntry(cachePrefix, url) return None return cachedname
def testCopyFileObjDigest(self): tmpDir = tempfile.mkdtemp() try: buf = 'test data' # rpepare source and destination files srcFn = os.path.join(tmpDir, 'srcfile') destFn = os.path.join(tmpDir, 'destfile') open(srcFn, 'w').write(buf) src = open(srcFn) dest = open(destFn, 'w') # filter the digest through copyfileobj sha1 = digestlib.sha1() util.copyfileobj(src, dest, digest=sha1, sizeLimit=len(buf)) res = sha1.hexdigest() # now compare the resulting hash to reference data sha1 = digestlib.sha1() sha1.update(buf) ref = sha1.hexdigest() self.assertEquals(ref, res) finally: util.rmtree(tmpDir)
def restore(self, fileContents, root, target, journal=None, sha1=None, nameLookup=True, **kwargs): keepTempfile = kwargs.get('keepTempfile', False) if fileContents != None: # this is first to let us copy the contents of a file # onto itself; the unlink helps that to work src = fileContents.get() inFd = None if fileContents.isCompressed(): if hasattr(src, '_fdInfo'): # inFd is None if we can't figure this information out # (for _LazyFile for instance) (inFd, inStart, inSize) = src._fdInfo() else: src = gzip.GzipFile(mode="r", fileobj=src) name = os.path.basename(target) path = os.path.dirname(target) if not os.path.isdir(path): util.mkdirChain(path) if inFd is not None: if keepTempfile: tmpfd, destTarget = tempfile.mkstemp(name, '.ct', path) os.close(tmpfd) destName = os.path.basename(destTarget) else: destName, destTarget = name, target actualSha1 = util.sha1Uncompress((inFd, inStart, inSize), path, destName, destTarget) if keepTempfile: # Set up the second temp file here. This makes # sure we get through the next if branch. inFd = None src = file(destTarget) elif keepTempfile: tmpfd, destTarget = tempfile.mkstemp(name, '.ct', path) f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f) f.close() src = file(destTarget) else: destTarget = target if inFd is None: tmpfd, tmpname = tempfile.mkstemp(name, '.ct', path) try: d = digestlib.sha1() f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f, digest=d) f.close() actualSha1 = d.digest() # would be nice if util could do this w/ a single # system call, but exists is better than an exception # when the file doesn't already exist if (os.path.exists(target) and stat.S_ISDIR(os.lstat(target).st_mode)): os.rmdir(target) os.rename(tmpname, target) except: # we've not renamed tmpname to target yet, we should # clean up instead of leaving temp files around os.unlink(tmpname) if keepTempfile: os.unlink(destTarget) raise if (sha1 is not None and sha1 != actualSha1): raise Sha1Exception(target) File.restore(self, root, target, journal=journal, nameLookup=nameLookup, **kwargs) else: destTarget = target File.restore(self, root, target, journal=journal, nameLookup=nameLookup, **kwargs) return destTarget
def verifySignatures(f, validKeys=None): """ Given an extended file, compute signatures """ f.seek(0) h = readHeader(f) # Cheap test first: verify MD5 sig sigmd5 = h.get(SIG_MD5, None) if sigmd5 is not None: f.seek(0) readSignatureHeader(f) # verify md5 digest md5 = digestlib.md5() util.copyfileobj(f, NullWriter(), digest=md5) if md5.digest() != sigmd5: raise MD5SignatureError( "The MD5 digest fails to verify: expected %s, got %s" % (sha1helper.md5ToString(sigmd5), md5.hexdigest())) # Don't bother if no gpg signature was present, or no valid keys were # presented if validKeys is None: return sigString = h.get(SIG_GPG, None) if sigString is None: return # Skip to immutable header region f.seek(0) readSignatureHeader(f) sig = openpgpfile.readSignature(sigString) keyId = sig.getSignerKeyId() matchingKeys = [x for x in validKeys if x.hasKeyId(keyId)] if not matchingKeys: raise PGPSignatureError("Signature generated with key %s does " "not match valid keys %s" % (keyId, ', '.join(x.getKeyId() for x in validKeys))) key = matchingKeys[0] # signature verification assumes a seekable stream and will seek to the # beginning; use a SeekableNestedFile size = h.getHeaderPlusPayloadSize() if size is None: pos = f.tell() f.seek(0, 2) size = f.tell() f.seek(pos, 0) snf = None if hasattr(f, 'pread'): extFile = f elif hasattr(f, 'name'): extFile = util.ExtendedFile(f.name, buffering=False) else: # worst case scenario, we slurp everything in memory extFile = util.ExtendedStringIO(f.read()) snf = extFile if snf is None: snf = util.SeekableNestedFile(extFile, start=f.tell(), size=size) try: sig.verifyDocument(key.getCryptoKey(), snf) except openpgpfile.SignatureError: raise PGPSignatureError
def __init__(self, sourceTrove, repos, cfg, versionStr=None, labelPath=None, ignoreInstalled=False, filterVersions=False, parentDir=None, defaultToLatest=False, buildFlavor=None, db=None, overrides=None, getFileFunction=None, branch=None): self.recipes = {} if getFileFunction is None: getFileFunction = lambda repos, fileId, fileVersion, path: \ repos.getFileContents([ (fileId, fileVersion) ])[0].get() name = sourceTrove.getName().split(':')[0] recipePath = name + '.recipe' match = self.findFileByPath(sourceTrove, recipePath) if not match: # this is just missing the recipe; we need it raise builderrors.RecipeFileError( "version %s of %s does not " "contain %s" % (sourceTrove.getName(), sourceTrove.getVersion().asString(), recipePath)) (fd, recipeFile) = tempfile.mkstemp(".recipe", 'temp-%s-' % name, dir=cfg.tmpDir) outF = os.fdopen(fd, "w") inF = getFileFunction(repos, match[0], match[1], recipePath) util.copyfileobj(inF, outF) del inF outF.close() del outF if branch is None: branch = sourceTrove.getVersion().branch() try: loadrecipe.RecipeLoader.__init__( self, recipeFile, cfg, repos, sourceTrove.getName(), branch=branch, ignoreInstalled=ignoreInstalled, directory=parentDir, buildFlavor=buildFlavor, db=db, overrides=overrides) finally: os.unlink(recipeFile) self.recipe._trove = sourceTrove.copy()
def digestWithProgress(fobj, digest, callback): source = StreamWithProgress(fobj, callback) return copyfileobj(source, Sink, digest=digest)
f.seek(0) tmp.write(' ') tmp.seek(0) try: rpmhelper.readHeader(tmp) except IOError, e: assert (str(e) == 'file size does not match size specified by ' 'header') else: assert (0) # change a byte in the header. the offset we write to here happens # to work for basesystem-8.0-2.src.rpm; if that file changes this # offset needs to change too tmp = tempfile.TemporaryFile() util.copyfileobj(f, tmp) tmp.seek(2000) tmp.write('X') tmp.seek(0) try: rpmhelper.readHeader(tmp) except IOError, e: assert (str(e) == "bad header sha1") else: assert (0) def testFilelessRpm(self): # Test that reading the paths for an rpm that has no files still works f = open(resources.get_archive() + "/fileless-0.1-1.noarch.rpm") h = rpmhelper.readHeader(f) self.assertEqual(list(h.paths()), [])