def _keys(self, items, args, kwargs): if kwargs is None: kwargs = () else: kwargs = tuple(sorted(kwargs.items())) common = digestlib.sha1(str(args + kwargs)).digest() return [digestlib.sha1(common + str(x)).hexdigest() for x in items]
def _create(self, certs, purpose, desc, issuer=None, common=None): """Create and store one certificate.""" if purpose in certs: return cu = self.db.cursor() subject = X509.X509_Name() subject.O = desc subject.OU = 'Created at ' + time.strftime('%F %T%z') if common is not None: subject.CN = common issuer_pkey = issuer_subject = issuer_fingerprint = serial = None if issuer is None: isCA = True else: isCA = False issuer_x509 = X509.load_cert_string(issuer[0]) issuer_pkey = EVP.load_key_string(issuer[1]) issuer_subject = issuer_x509.get_subject() issuer_fingerprint = digestlib.sha1( issuer_x509.as_der()).hexdigest() cu.execute("""UPDATE pki_certificates SET ca_serial_index = ca_serial_index + 1 WHERE fingerprint = %s RETURNING ca_serial_index """, (issuer_fingerprint,)) serial, = cu.fetchone() # Create certificates with a 'not before' date 1 day in the past, just # in case initial setup sets the clock backwards. rsa, x509 = gencert.new_cert(KEY_LENGTH, subject, EXPIRY, issuer=issuer_subject, issuer_evp=issuer_pkey, isCA=isCA, serial=serial, timestamp_offset=-86400) fingerprint = digestlib.sha1(x509.as_der()).hexdigest() pkey_pem = rsa.as_pem(None) x509_pem = x509.as_pem() cu.execute("""INSERT INTO pki_certificates ( fingerprint, purpose, is_ca, x509_pem, pkey_pem, issuer_fingerprint, ca_serial_index, time_issued, time_expired ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)""", (fingerprint, purpose, isCA, x509_pem, pkey_pem, issuer_fingerprint, 0, str(x509.get_not_before()), str(x509.get_not_after()), )) log.info("Created certificate %s for purpose %r%s%s", fingerprint, purpose, (issuer_fingerprint and (" (issuer %s)" % issuer_fingerprint) or ""), self.dry_run and " (dry run)" or "") certs[purpose] = x509_pem, pkey_pem
def incrementCount(self, hash, fileObj=None, precompressed=True): """ Increments the count by one. If it becomes one (the file is new), the contents of fileObj are stored into that path. """ if len(hash) != 40: hash = sha1helper.sha1ToString(hash) cu = self.db.cursor() cu.execute("SELECT COUNT(*) FROM DataStore WHERE hash=?", hash) exists = cu.next()[0] if exists: cu.execute("UPDATE DataStore SET count=count+1 WHERE hash=?", hash) else: if precompressed: # it's precompressed as a gzip stream, and we need a # zlib stream. just decompress it. gzObj = gzip.GzipFile(mode="r", fileobj=fileObj) rawData = gzObj.read() del gzObj else: rawData = fileObj.read() data = zlib.compress(rawData) digest = digestlib.sha1() digest.update(rawData) if digest.hexdigest() != hash: raise errors.IntegrityError cu.execute("INSERT INTO DataStore VALUES(?, 1, ?)", hash, data)
def _writeFile(cls, fileObj, outFds, precompressed, computeSha1): if precompressed and hasattr(fileObj, '_fdInfo'): (fd, start, size) = fileObj._fdInfo() pid = os.getpid() realHash = digest_uncompress.sha1Copy((fd, start, size), outFds) for x in outFds: cls._fchmod(x) os.close(x) return realHash else: for fd in outFds: outFileObj = os.fdopen(fd, "w") contentSha1 = digestlib.sha1() if precompressed and computeSha1: tee = Tee(fileObj, outFileObj) uncompObj = gzip.GzipFile(mode = "r", fileobj = tee) s = uncompObj.read(128 * 1024) while s: contentSha1.update(s) s = uncompObj.read(128 * 1024) uncompObj.close() elif precompressed: util.copyfileobj(fileObj, outFileObj) else: dest = gzip.GzipFile(mode = "w", fileobj = outFileObj) util.copyfileobj(fileObj, dest, digest = contentSha1) dest.close() # this closes tmpFd for us cls._fchmod(fd) outFileObj.close() fileObj.seek(0) return contentSha1.digest()
def restore(self, fileContents, root, target, journal=None, sha1=None, nameLookup=True, **kwargs): keepTempfile = kwargs.get('keepTempfile', False) destTarget = target if fileContents is not None: # this is first to let us copy the contents of a file # onto itself; the unlink helps that to work src = fileContents.get() inFd = None if fileContents.isCompressed() and hasattr(src, '_fdInfo'): # inFd is None if we can't figure this information out # (for _LazyFile for instance) (inFd, inStart, inSize) = src._fdInfo() path, name = os.path.split(target) if not os.path.isdir(path): util.mkdirChain(path) # Uncompress to a temporary file, using the accelerated # implementation if possible. if inFd is not None and util.sha1Uncompress is not None: actualSha1, tmpname = util.sha1Uncompress( inFd, inStart, inSize, path, name) else: if fileContents.isCompressed(): src = gzip.GzipFile(mode='r', fileobj=src) tmpfd, tmpname = tempfile.mkstemp(name, '.ct', path) try: d = digestlib.sha1() f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f, digest=d) f.close() actualSha1 = d.digest() except: os.unlink(tmpname) raise if keepTempfile: # Make a hardlink "copy" for the caller to use destTarget = tmpname + '.ptr' os.link(tmpname, destTarget) try: os.rename(tmpname, target) except OSError, err: if err.args[0] != errno.EISDIR: raise os.rmdir(target) os.rename(tmpname, target) if (sha1 is not None and sha1 != actualSha1): raise Sha1Exception(target)
def incrementCount(self, hash, fileObj = None, precompressed = True): """ Increments the count by one. If it becomes one (the file is new), the contents of fileObj are stored into that path. """ if len(hash) != 40: hash = sha1helper.sha1ToString(hash) cu = self.db.cursor() cu.execute("SELECT COUNT(*) FROM DataStore WHERE hash=?", hash) exists = cu.next()[0] if exists: cu.execute("UPDATE DataStore SET count=count+1 WHERE hash=?", hash) else: if precompressed: # it's precompressed as a gzip stream, and we need a # zlib stream. just decompress it. gzObj = gzip.GzipFile(mode = "r", fileobj = fileObj) rawData = gzObj.read() del gzObj else: rawData = fileObj.read() data = zlib.compress(rawData) digest = digestlib.sha1() digest.update(rawData) if digest.hexdigest() != hash: raise errors.IntegrityError cu.execute("INSERT INTO DataStore VALUES(?, 1, ?)", hash, data)
def _writeFile(cls, fileObj, outFds, precompressed, computeSha1): if precompressed and hasattr(fileObj, '_fdInfo'): (fd, start, size) = fileObj._fdInfo() pid = os.getpid() realHash = digest_uncompress.sha1Copy((fd, start, size), outFds) for x in outFds: cls._fchmod(x) os.close(x) return realHash else: for fd in outFds: outFileObj = os.fdopen(fd, "w") contentSha1 = digestlib.sha1() if precompressed and computeSha1: tee = Tee(fileObj, outFileObj) uncompObj = gzip.GzipFile(mode="r", fileobj=tee) s = uncompObj.read(128 * 1024) while s: contentSha1.update(s) s = uncompObj.read(128 * 1024) uncompObj.close() elif precompressed: util.copyfileobj(fileObj, outFileObj) else: dest = gzip.GzipFile(mode="w", fileobj=outFileObj) util.copyfileobj(fileObj, dest, digest=contentSha1) dest.close() # this closes tmpFd for us cls._fchmod(fd) outFileObj.close() fileObj.seek(0) return contentSha1.digest()
def apply(self, justDatabase=False, noScripts=False, capsuleChangeSet=None): if capsuleChangeSet: # Previous jobs will have moved the pointer in the auxilliary # changeset, so reset it at the start of each job. capsuleChangeSet.reset() tmpDir = os.path.join(self.root, 'var/tmp') if not os.path.isdir(tmpDir): # For empty roots or roots that are not systems (e.g. source # checkouts), just put capsules in the root directory. tmpDir = self.root fileDict = {} for kind, obj in sorted(self.capsuleClasses.items()): fileDict.update( dict(((x[0], x[2], x[3]), x[1]) for x in obj._filesNeeded())) try: for ((pathId, fileId, sha1), path) in sorted(fileDict.items()): tmpfd, tmpname = tempfile.mkstemp(dir=tmpDir, prefix=path, suffix='.conary') fType, fContents = self.changeSet.getFileContents( pathId, fileId) if (fType == changeset.ChangedFileTypes.hldr): if (capsuleChangeSet): try: result = capsuleChangeSet.getFileContents( pathId, fileId) fObj = result[1].get() except KeyError: raise errors.MissingRollbackCapsule( 'Cannot find ' 'RPM %s to perform local rollback' % path) else: fObj = fContents.get() d = digestlib.sha1() util.copyfileobj(fObj, os.fdopen(tmpfd, "w"), digest=d) actualSha1 = d.digest() if actualSha1 != sha1: raise files.Sha1Exception(path) # tmpfd is closed when the file object created by os.fdopen # disappears fileDict[(pathId, fileId)] = tmpname for kind, obj in sorted(self.capsuleClasses.items()): obj.apply(fileDict, justDatabase=justDatabase, noScripts=noScripts) finally: for tmpPath in fileDict.values(): try: os.unlink(tmpPath) except: pass
def oldtest(self): cpioPath = self._createCpio() sha1sum = digestlib.sha1(file(cpioPath).read()).hexdigest() resultFilePath = os.path.join(self.workDir, 'result.cpio') # Use a variety of sizes, to try to come up with different chunking # solutions for bufferSize in [1001, 1003, 3001]: f = file(resultFilePath, "w") src = cpiostream.CpioStream(file(cpioPath)) while 1: buf = src.read(bufferSize) if not buf: break f.write(buf) f.close() nsha1sum = digestlib.sha1(file(resultFilePath).read()).hexdigest() self.assertEqual(nsha1sum, sha1sum)
def testExpansion(self): cpioPath = self._createCpio() target = self.workDir + '/root' expander = cpiostream.CpioExploder(file(cpioPath)) expander.explode(target) sha1sum = digestlib.sha1(file( target + '/usr/lib/perl5/5.10.0/Archive/Tar.pm').read()).hexdigest() self.assertEquals(sha1sum, 'cbe78d8a0d26a86436e4fc56f8581ffd3db4bd83') shutil.rmtree(self.workDir) os.mkdir(self.workDir) cpioPath = self._createCpio(rpmName = 'simple-1.1-1.i386.rpm') expander = cpiostream.CpioExploder(file(cpioPath)) expander.explode(target) assert(os.path.isdir(target + '/dir')) sha1sum = digestlib.sha1(file(target + '/normal').read()).hexdigest() self.assertEquals(sha1sum, '5662cdf7d378e7505362c59239f73107b6edf1d3')
def getFingerprint(self): d = digestlib.sha1() for fplist in [ self.jobFingerprints, self.bootstrapFingerprints, self.crossFingerprints, ]: d.update(''.join(sorted(fplist)) + '\0\0') d.update('\0'.join(x.freeze() for x in sorted(self.rpmRequirements))) return d.digest()
def testPrelink(self): user = pwd.getpwuid(os.getuid()).pw_name group = grp.getgrgid(os.getgid()).gr_name archivePath = resources.get_archive() self.addComponent('test:foo=1', fileContents=[ ('/prelinktest', rephelp.RegularFile( contents=open(archivePath + '/prelinktest'), owner=user, group=group, mode=0755)), ('/prelinktest-orig', rephelp.RegularFile( contents=open(archivePath + '/prelinktest'), owner=user, group=group, mode=0755)) ]) self.updatePkg('test:foo=1') db = database.Database(self.rootDir, self.cfg.dbPath) rc, str = self.captureOutput(verify.verify, ['test:foo'], db, self.cfg) self.assertEquals(str, '') binary = self.rootDir + '/prelinktest' # the test suite can't set the mtime on the file; we'll preserve # it ourself sb = os.stat(binary) os.system("cp %s/prelinktest-prelinked %s" % (archivePath, binary)) os.utime(binary, (sb.st_atime, sb.st_mtime)) self.assertEquals(files.PRELINK_CMD, ('/usr/sbin/prelink', )) oldCmd = files.PRELINK_CMD try: files.PRELINK_CMD = (archivePath + '/prelink', ) files._havePrelink = None rc, str = self.captureOutput(verify.verify, ['test:foo'], db, self.cfg, forceHashCheck=True) self.assertEquals(str, '') # Also verify a path used by addCapsule f, nlinks, devino = files.FileFromFilesystem(binary, pathId='\0' * 16, inodeInfo=True) self.assertEquals( digestlib.sha1(open(binary).read()).hexdigest(), '1114f3a978b60d76d7618dc43aaf207bc999f997') self.assertEquals(f.contents.sha1().encode('hex'), '23ad3a2c940a30809b68a5b8a13392196004efab') finally: files.PRELINK_CMD = oldCmd files._havePrelink = None
def restore(self, fileContents, root, target, journal=None, sha1 = None, nameLookup=True, **kwargs): keepTempfile = kwargs.get('keepTempfile', False) destTarget = target if fileContents is not None: # this is first to let us copy the contents of a file # onto itself; the unlink helps that to work src = fileContents.get() inFd = None if fileContents.isCompressed() and hasattr(src, '_fdInfo'): # inFd is None if we can't figure this information out # (for _LazyFile for instance) (inFd, inStart, inSize) = src._fdInfo() path, name = os.path.split(target) if not os.path.isdir(path): util.mkdirChain(path) # Uncompress to a temporary file, using the accelerated # implementation if possible. if inFd is not None and util.sha1Uncompress is not None: actualSha1, tmpname = util.sha1Uncompress( inFd, inStart, inSize, path, name) else: if fileContents.isCompressed(): src = gzip.GzipFile(mode='r', fileobj=src) tmpfd, tmpname = tempfile.mkstemp(name, '.ct', path) try: d = digestlib.sha1() f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f, digest = d) f.close() actualSha1 = d.digest() except: os.unlink(tmpname) raise if keepTempfile: # Make a hardlink "copy" for the caller to use destTarget = tmpname + '.ptr' os.link(tmpname, destTarget) try: os.rename(tmpname, target) except OSError, err: if err.args[0] != errno.EISDIR: raise os.rmdir(target) os.rename(tmpname, target) if (sha1 is not None and sha1 != actualSha1): raise Sha1Exception(target)
def getElementTree(self, *args, **kwargs): eltree = xmllib.BaseNode.getElementTree(self, *args, **kwargs) if '_xmlNodeHash' not in self.__slots__ or self._xmlNodeHash is not None: return eltree # Compute the checksum csum = digestlib.sha1() csum.update(xmllib.etree.tostring(eltree, pretty_print = False, xml_declaration = False, encoding = 'UTF-8')) self._xmlNodeHash = csum.hexdigest() eltree.attrib['xmlNodeHash'] = self._xmlNodeHash return eltree
def apply(self, justDatabase = False, noScripts = False, capsuleChangeSet = None): if capsuleChangeSet: # Previous jobs will have moved the pointer in the auxilliary # changeset, so reset it at the start of each job. capsuleChangeSet.reset() tmpDir = os.path.join(self.root, 'var/tmp') if not os.path.isdir(tmpDir): # For empty roots or roots that are not systems (e.g. source # checkouts), just put capsules in the root directory. tmpDir = self.root fileDict = {} for kind, obj in sorted(self.capsuleClasses.items()): fileDict.update( dict(((x[0], x[2], x[3]), x[1]) for x in obj._filesNeeded())) try: for ((pathId, fileId, sha1), path) in sorted(fileDict.items()): tmpfd, tmpname = tempfile.mkstemp(dir=tmpDir, prefix=path, suffix='.conary') fType, fContents = self.changeSet.getFileContents(pathId, fileId) if (fType == changeset.ChangedFileTypes.hldr): if (capsuleChangeSet): try: result = capsuleChangeSet.getFileContents(pathId, fileId) fObj = result[1].get() except KeyError: raise errors.MissingRollbackCapsule('Cannot find ' 'RPM %s to perform local rollback' % path) else: fObj = fContents.get() d = digestlib.sha1() util.copyfileobj(fObj, os.fdopen(tmpfd, "w"), digest = d) actualSha1 = d.digest() if actualSha1 != sha1: raise files.Sha1Exception(path) # tmpfd is closed when the file object created by os.fdopen # disappears fileDict[(pathId, fileId)] = tmpname for kind, obj in sorted(self.capsuleClasses.items()): obj.apply(fileDict, justDatabase = justDatabase, noScripts = noScripts) finally: for tmpPath in fileDict.values(): try: os.unlink(tmpPath) except: pass
def getJobHash(self): # Disqualify anything other than a simple, isolated build. if self.inCycle or self.builtTroves or self.crossTroves: return None # Hash all the inputs to the resolver so that the result can be cached. inputs = [ '\0'.join(sorted(self.trove.getBuildRequirements())), '\0'.join(sorted(self.trove.getCrossRequirements())), '\0'.join(str(x) for x in self.buildCfg.flavor), '\1'.join('\0'.join(sorted(str(y) for y in x)) for x in self.buildCfg.resolveTroveTups), ] return digestlib.sha1('\2'.join(inputs)).hexdigest()
def testPrelink(self): user = pwd.getpwuid(os.getuid()).pw_name group = grp.getgrgid(os.getgid()).gr_name archivePath = resources.get_archive() self.addComponent( "test:foo=1", fileContents=[ ( "/prelinktest", rephelp.RegularFile( contents=open(archivePath + "/prelinktest"), owner=user, group=group, mode=0755 ), ), ( "/prelinktest-orig", rephelp.RegularFile( contents=open(archivePath + "/prelinktest"), owner=user, group=group, mode=0755 ), ), ], ) self.updatePkg("test:foo=1") db = database.Database(self.rootDir, self.cfg.dbPath) rc, str = self.captureOutput(verify.verify, ["test:foo"], db, self.cfg) self.assertEquals(str, "") binary = self.rootDir + "/prelinktest" # the test suite can't set the mtime on the file; we'll preserve # it ourself sb = os.stat(binary) os.system("cp %s/prelinktest-prelinked %s" % (archivePath, binary)) os.utime(binary, (sb.st_atime, sb.st_mtime)) self.assertEquals(files.PRELINK_CMD, ("/usr/sbin/prelink",)) oldCmd = files.PRELINK_CMD try: files.PRELINK_CMD = (archivePath + "/prelink",) files._havePrelink = None rc, str = self.captureOutput(verify.verify, ["test:foo"], db, self.cfg, forceHashCheck=True) self.assertEquals(str, "") # Also verify a path used by addCapsule f, nlinks, devino = files.FileFromFilesystem(binary, pathId="\0" * 16, inodeInfo=True) self.assertEquals( digestlib.sha1(open(binary).read()).hexdigest(), "1114f3a978b60d76d7618dc43aaf207bc999f997" ) self.assertEquals(f.contents.sha1().encode("hex"), "23ad3a2c940a30809b68a5b8a13392196004efab") finally: files.PRELINK_CMD = oldCmd files._havePrelink = None
def _generate(self): self._log.info("Generating template %s from trove %s=%s[%s]", self._hash, *self._troveTup) self._installContents(self._contentsDir, [self._troveTup]) # Copy "unified" directly into the output. os.mkdir(self._outputDir) util.copytree(self._contentsDir + '/unified', self._outputDir + '/') # Process the MANIFEST file. for line in open(self._contentsDir + '/MANIFEST'): line = line.rstrip() if not line or line[0] == '#': continue args = line.rstrip().split(',') command = args.pop(0) commandFunc = getattr(self, '_DO_' + command, None) if not commandFunc: raise RuntimeError("Unknown command %r in MANIFEST" % (command,)) commandFunc(args) # Archive the results. digest = digestlib.sha1() outFile = util.AtomicFile(self._outputPath) proc = call(['/bin/tar', '-cC', self._outputDir, '.'], stdout=subprocess.PIPE, captureOutput=False, wait=False) util.copyfileobj(proc.stdout, outFile, digest=digest) proc.wait() # Write metadata. metaFile = util.AtomicFile(self._outputPath + '.metadata') cPickle.dump({ 'sha1sum': digest.hexdigest(), 'trovespec': '%s=%s[%s]' % self._troveTup, 'kernel': (self._kernelTup and ('%s=%s[%s]' % self._kernelTup) or '<none>'), # Right now, we are going to hardcode this to an older version # of Netclient Protocol to hint to the Conary installed on the # jobslave to generate old filecontainers that are compatible # with all versions of Conary. (See RBL-1552.) 'netclient_protocol_version': '38', }, metaFile) metaFile.commit() outFile.commit() self._log.info("Template %s created", self._hash)
def getElementTree(self, *args, **kwargs): eltree = xmllib.BaseNode.getElementTree(self, *args, **kwargs) if '_xmlNodeHash' not in self.__slots__ or self._xmlNodeHash is not None: return eltree # Compute the checksum csum = digestlib.sha1() csum.update( xmllib.etree.tostring(eltree, pretty_print=False, xml_declaration=False, encoding='UTF-8')) self._xmlNodeHash = csum.hexdigest() eltree.attrib['xmlNodeHash'] = self._xmlNodeHash return eltree
def read(self): instanceId = self.ec2InstanceId if instanceId is not None: sha = digestlib.sha1(instanceId) self._uuid = GeneratedUuid.asString(sha.digest()[:16]) else: dmidecodeUuid = self._getDmidecodeUuid().lower() self._uuid = dmidecodeUuid if os.path.exists(self.uuidFile): persistedUuid = self._readFile(self.uuidFile) if persistedUuid.lower() != self._uuid: self._writeDmidecodeUuid(self._uuid) else: self._writeDmidecodeUuid(self._uuid)
def testCopyFileObjDigest(self): tmpDir = tempfile.mkdtemp() try: buf = 'test data' # rpepare source and destination files srcFn = os.path.join(tmpDir, 'srcfile') destFn = os.path.join(tmpDir, 'destfile') open(srcFn, 'w').write(buf) src = open(srcFn) dest = open(destFn, 'w') # filter the digest through copyfileobj sha1 = digestlib.sha1() util.copyfileobj(src, dest, digest=sha1, sizeLimit=len(buf)) res = sha1.hexdigest() # now compare the resulting hash to reference data sha1 = digestlib.sha1() sha1.update(buf) ref = sha1.hexdigest() self.assertEquals(ref, res) finally: util.rmtree(tmpDir)
def testCopyFileObjDigest(self): tmpDir = tempfile.mkdtemp() try: buf = "test data" # rpepare source and destination files srcFn = os.path.join(tmpDir, "srcfile") destFn = os.path.join(tmpDir, "destfile") open(srcFn, "w").write(buf) src = open(srcFn) dest = open(destFn, "w") # filter the digest through copyfileobj sha1 = digestlib.sha1() util.copyfileobj(src, dest, digest=sha1, sizeLimit=len(buf)) res = sha1.hexdigest() # now compare the resulting hash to reference data sha1 = digestlib.sha1() sha1.update(buf) ref = sha1.hexdigest() self.assertEquals(ref, res) finally: util.rmtree(tmpDir)
def sha1FileBin(path): oldmode = None mode = os.lstat(path)[stat.ST_MODE] if (mode & 0400) != 0400: oldmode = mode os.chmod(path, mode | 0400) fd = os.open(path, os.O_RDONLY) if oldmode is not None: os.chmod(path, oldmode) m = digestlib.sha1() buf = os.read(fd, 40960) while len(buf): m.update(buf) buf = os.read(fd, 40960) os.close(fd) return m.digest()
def setImageFiles(self, db, hostname, imageId, imageFiles=None): if imageFiles is None: digest = sha1() digest.update(str(imageId)) digest = digest.hexdigest() imageFiles = models.ImageFileList(files=[models.ImageFile( title='Image File %s' % imageId, size=1024 * imageId, sha1=digest, fileName='imagefile_%s.iso' % imageId, )]) for item in imageFiles.files: path = '%s/%s/%s/%s' % (self.mintCfg.imagesPath, hostname, imageId, item.fileName) util.mkdirChain(os.path.dirname(path)) open(path, 'w').write('image data') db.imageMgr.setFilesForImage(hostname, imageId, imageFiles)
def get(self, pathId): f = self.fileClass(pathId) f.inode = files.InodeStream(self.perms & 07777, self.mtime, self.owner, self.group) self._touchupFileStream(f) if self.needSha1: sha1 = digestlib.sha1() contents = self.contents.get() devnull = open(os.devnull, "w") util.copyfileobj(contents, devnull, digest=sha1) devnull.close() f.contents = files.RegularFileStream() f.contents.size.set(contents.tell()) f.contents.sha1.set(sha1.digest()) f.provides.set(self.provides) f.requires.set(self.requires) f.flavor.set(self.flavor) for tag in self.tags: f.tags.set(tag) return f
def get(self, pathId): f = self.fileClass(pathId) f.inode = files.InodeStream(self.perms & 07777, self.mtime, self.owner, self.group) self._touchupFileStream(f) if self.needSha1: sha1 = digestlib.sha1() contents = self.contents.get() devnull = open(os.devnull, 'w') util.copyfileobj(contents, devnull, digest=sha1) devnull.close() f.contents = files.RegularFileStream() f.contents.size.set(contents.tell()) f.contents.sha1.set(sha1.digest()) f.provides.set(self.provides) f.requires.set(self.requires) f.flavor.set(self.flavor) for tag in self.tags: f.tags.set(tag) return f
def newCollection(self, kvdict): cu = self._db.cursor() jobType = kvdict['type'] jobTypeId = self.jobTypes[jobType] jobStateId = self.jobStates['Queued'] now = time.time() created = kvdict.setdefault('created', now) modified = kvdict.setdefault('modified', now) ttl = kvdict.setdefault('ttl', None) if ttl: expiration = modified + ttl else: expiration = None if 'jobUuid' not in kvdict: jobUuid = digestlib.sha1(file("/dev/urandom").read(16)).hexdigest() kvdict['jobUuid'] = jobUuid jobUuid = kvdict['jobUuid'] extraCreateArgs = self._extraCreateArgs(kvdict) args = (jobUuid, jobTypeId, jobStateId, self._db.auth.userId, created, modified, expiration, ttl) + extraCreateArgs assert len(self.extra_fields_insert) == len(extraCreateArgs) if self.extra_fields_insert: extra_fields = ", " + ', '.join(self.extra_fields_insert) extra_bind_args = ", " + ', '.join('?' for x in self.extra_fields_insert) else: extra_fields = "" extra_bind_args = "" sql = """ INSERT INTO jobs (job_uuid, job_type_id, job_state_id, created_by, created, modified, expiration, ttl%(extra_fields)s) VALUES (?, ?, ?, ?, ?, ?, ?, ?%(extra_bind_args)s)""" % dict( extra_fields = extra_fields, extra_bind_args = extra_bind_args) cu.execute(sql, *args) jobId = cu.lastid() self._postNewCollection(jobId, kvdict) return jobId
def uploadBuild(self, auth): method = self.req.method.upper() if method != "PUT": raise web_exc.HTTPMethodNotAllowed(allow='PUT') client = shimclient.ShimMintClient(self.cfg, (self.cfg.authUser, self.cfg.authPass), self.db) buildId, fileName = self.req.path_info.split("/")[-2:] build = client.getBuild(int(buildId)) project = client.getProject(build.projectId) # make sure the hash we receive from the slave matches # the hash we gave the slave in the first place. # this prevents slaves from overwriting arbitrary files # in the finished images directory. if not auth.admin: outputToken = self.req.headers.get('X-rBuilder-OutputToken') if outputToken != build.getDataValue('outputToken', validate = False): raise web_exc.HTTPForbidden() targetFn = os.path.join(self.cfg.imagesPath, project.hostname, str(buildId), fileName) util.mkdirChain(os.path.dirname(targetFn)) fObj = AtomicFile(targetFn, 'wb+', prefix='img-', suffix='.tmp') ctx = digestlib.sha1() inFile = None if 'x-uploaded-file' in self.req.headers: # The frontend proxy has already saved the request body to a # temporary location, so first try to rename it into place. try: os.rename(self.req.headers['x-uploaded-file'], fObj.name) except OSError, err: if err.errno != errno.EXDEV: raise # Upload dir is on a different filesystem. inFile = open(self.req.headers['x-uploaded-file'], 'rb')
def specHash(troveTups, buildTimes=None): """ Create a unique identifier for the troves C{troveTups}. """ if buildTimes: assert len(troveTups) == len(buildTimes) troveTups = zip(troveTups, buildTimes) else: troveTups = [(x, None) for x in troveTups] items = [] for (name, version, flavor), buildTime in sorted(troveTups): items.append(name) if buildTime: items.append(version.trailingRevision().version) items.append(long(buildTime)) elif version.trailingRevision().timeStamp: items.append(version.freeze()) else: items.append(version.asString()) items.append(flavor.freeze()) items.append('') return digestlib.sha1('\0'.join(str(x) for x in items)).hexdigest()
def get(self, fileName, computeShaDigest = False): """ Download a file from the repository. @param fileName: relative path to file @type fileName: string @return open file instance """ fobj = self._getTempFileObject() realUrl = self._getRealUrl(fileName) inf = self._opener.open(realUrl) if computeShaDigest: dig = digestlib.sha1() else: dig = None util.copyfileobj(inf, fobj, digest = dig) fobj.seek(0) if not os.path.basename(fileName).endswith('.gz'): return self.FileWrapper.create(fobj, dig) return self.FileWrapper.create(gzip.GzipFile(fileobj=fobj, mode="r"), dig)
def __init__(self, fobj): self.fobj = fobj self.digest = digestlib.sha1()
def sha1String(buf): m = digestlib.sha1() m.update(buf) return m.digest()
def restore(self, fileContents, root, target, journal=None, sha1=None, nameLookup=True, **kwargs): keepTempfile = kwargs.get('keepTempfile', False) if fileContents != None: # this is first to let us copy the contents of a file # onto itself; the unlink helps that to work src = fileContents.get() inFd = None if fileContents.isCompressed(): if hasattr(src, '_fdInfo'): # inFd is None if we can't figure this information out # (for _LazyFile for instance) (inFd, inStart, inSize) = src._fdInfo() else: src = gzip.GzipFile(mode="r", fileobj=src) name = os.path.basename(target) path = os.path.dirname(target) if not os.path.isdir(path): util.mkdirChain(path) if inFd is not None: if keepTempfile: tmpfd, destTarget = tempfile.mkstemp(name, '.ct', path) os.close(tmpfd) destName = os.path.basename(destTarget) else: destName, destTarget = name, target actualSha1 = util.sha1Uncompress((inFd, inStart, inSize), path, destName, destTarget) if keepTempfile: # Set up the second temp file here. This makes # sure we get through the next if branch. inFd = None src = file(destTarget) elif keepTempfile: tmpfd, destTarget = tempfile.mkstemp(name, '.ct', path) f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f) f.close() src = file(destTarget) else: destTarget = target if inFd is None: tmpfd, tmpname = tempfile.mkstemp(name, '.ct', path) try: d = digestlib.sha1() f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f, digest=d) f.close() actualSha1 = d.digest() # would be nice if util could do this w/ a single # system call, but exists is better than an exception # when the file doesn't already exist if (os.path.exists(target) and stat.S_ISDIR(os.lstat(target).st_mode)): os.rmdir(target) os.rename(tmpname, target) except: # we've not renamed tmpname to target yet, we should # clean up instead of leaving temp files around os.unlink(tmpname) if keepTempfile: os.unlink(destTarget) raise if (sha1 is not None and sha1 != actualSha1): raise Sha1Exception(target) File.restore(self, root, target, journal=journal, nameLookup=nameLookup, **kwargs) else: destTarget = target File.restore(self, root, target, journal=journal, nameLookup=nameLookup, **kwargs) return destTarget
def FileFromFilesystem(path, pathId, possibleMatch = None, inodeInfo = False, assumeRoot=False, statBuf=None, sha1FailOk=False): if statBuf: s = statBuf else: s = os.lstat(path) global userCache, groupCache, _havePrelink if assumeRoot: owner = 'root' group = 'root' elif isinstance(s.st_uid, basestring): # Already stringified -- some capsule code will fabricate a stat result # from e.g. a RPM header owner = s.st_uid group = s.st_gid else: # + is not a valid char in user/group names; if the uid is not mapped # to a user, prepend it with + and store it as a string try: owner = userCache.lookupId('/', s.st_uid) except KeyError: owner = '+%d' % s.st_uid try: group = groupCache.lookupId('/', s.st_gid) except KeyError: group = '+%d' % s.st_gid needsSha1 = 0 inode = InodeStream(s.st_mode & 07777, s.st_mtime, owner, group) if (stat.S_ISREG(s.st_mode)): f = RegularFile(pathId) needsSha1 = 1 elif (stat.S_ISLNK(s.st_mode)): f = SymbolicLink(pathId) if hasattr(s, 'linkto'): f.target.set(s.linkto) else: f.target.set(os.readlink(path)) elif (stat.S_ISDIR(s.st_mode)): f = Directory(pathId) elif (stat.S_ISSOCK(s.st_mode)): f = Socket(pathId) elif (stat.S_ISFIFO(s.st_mode)): f = NamedPipe(pathId) elif (stat.S_ISBLK(s.st_mode)): f = BlockDevice(pathId) f.devt.major.set(s.st_rdev >> 8) f.devt.minor.set(s.st_rdev & 0xff) elif (stat.S_ISCHR(s.st_mode)): f = CharacterDevice(pathId) f.devt.major.set(s.st_rdev >> 8) f.devt.minor.set(s.st_rdev & 0xff) else: raise FilesError("unsupported file type for %s" % path) f.inode = inode f.flags = FlagsStream(0) # assume we have a match if the FileMode and object type match if possibleMatch and (possibleMatch.__class__ == f.__class__) \ and f.inode == possibleMatch.inode \ and f.inode.mtime() == possibleMatch.inode.mtime() \ and (not s.st_size or (possibleMatch.hasContents and s.st_size == possibleMatch.contents.size())): f.flags.set(possibleMatch.flags()) return possibleMatch elif (possibleMatch and (isinstance(f, RegularFile) and isinstance(possibleMatch, RegularFile)) and (f.inode.isExecutable()) and f.inode.mtime() == possibleMatch.inode.mtime() and f.inode.owner == possibleMatch.inode.owner and f.inode.group == possibleMatch.inode.group and f.inode.perms == possibleMatch.inode.perms): # executable RegularFiles match even if there sizes are different # as long as everything else is the same; this is to stop size # changes from prelink from changing fileids return possibleMatch if needsSha1: f.contents = RegularFileStream() undoPrelink = False if _havePrelink != False and f.inode.isExecutable(): try: from conary.lib import elf if elf.prelinked(path): undoPrelink = True except: pass if undoPrelink and _havePrelink is None: _havePrelink = bool(os.access(PRELINK_CMD[0], os.X_OK)) if undoPrelink and _havePrelink: prelink = subprocess.Popen( PRELINK_CMD + ("-y", path), stdout = subprocess.PIPE, close_fds = True, shell = False) d = digestlib.sha1() content = prelink.stdout.read() size = 0 while content: d.update(content) size += len(content) content = prelink.stdout.read() prelink.wait() f.contents.size.set(size) sha1 = d.digest() else: try: sha1 = sha1helper.sha1FileBin(path) except OSError: if sha1FailOk: sha1 = sha1helper.sha1Empty else: raise f.contents.size.set(s.st_size) f.contents.sha1.set(sha1) if inodeInfo: return (f, s.st_nlink, (s.st_rdev, s.st_ino)) return f
def fingerprint(self): return digestlib.sha1(self.x509.as_der()).hexdigest()
def restore(self, fileContents, root, target, journal=None, sha1 = None, nameLookup=True, **kwargs): keepTempfile = kwargs.get('keepTempfile', False) if fileContents != None: # this is first to let us copy the contents of a file # onto itself; the unlink helps that to work src = fileContents.get() inFd = None if fileContents.isCompressed(): if hasattr(src, '_fdInfo'): # inFd is None if we can't figure this information out # (for _LazyFile for instance) (inFd, inStart, inSize) = src._fdInfo() else: src = gzip.GzipFile(mode = "r", fileobj = src) name = os.path.basename(target) path = os.path.dirname(target) if not os.path.isdir(path): util.mkdirChain(path) if inFd is not None: if keepTempfile: tmpfd, destTarget = tempfile.mkstemp(name, '.ct', path) os.close(tmpfd) destName = os.path.basename(destTarget) else: destName, destTarget = name, target actualSha1 = util.sha1Uncompress((inFd, inStart, inSize), path, destName, destTarget) if keepTempfile: # Set up the second temp file here. This makes # sure we get through the next if branch. inFd = None src = file(destTarget) elif keepTempfile: tmpfd, destTarget = tempfile.mkstemp(name, '.ct', path) f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f) f.close() src = file(destTarget) else: destTarget = target if inFd is None: tmpfd, tmpname = tempfile.mkstemp(name, '.ct', path) try: d = digestlib.sha1() f = os.fdopen(tmpfd, 'w') util.copyfileobj(src, f, digest = d) f.close() actualSha1 = d.digest() # would be nice if util could do this w/ a single # system call, but exists is better than an exception # when the file doesn't already exist if (os.path.exists(target) and stat.S_ISDIR(os.lstat(target).st_mode)): os.rmdir(target) os.rename(tmpname, target) except: # we've not renamed tmpname to target yet, we should # clean up instead of leaving temp files around os.unlink(tmpname) if keepTempfile: os.unlink(destTarget) raise if (sha1 is not None and sha1 != actualSha1): raise Sha1Exception(target) File.restore(self, root, target, journal=journal, nameLookup=nameLookup, **kwargs) else: destTarget = target File.restore(self, root, target, journal=journal, nameLookup=nameLookup, **kwargs) return destTarget
def createCertificate(self, purpose, desc, issuer=None, common=None): """Create and store one certificate. @param purpose: Machine-readable string identifying the purpose of this certificate. @param desc: Human-readable description to put into the certificate. @param issuer: Optional tuple C{(x509, pkey)} of issuer cert pair. @param common: Optional common name (hostname) for subject. """ # Don't let anyone else even read the table while we work. We wouldn't # want two processes to read in the same CA serial index, generate # different certificates using the same serial, write them out to disk, # then blow up when they try to commit. cu = self.db.cursor() cu.execute("LOCK TABLE pki_certificates") subject = X509.X509_Name() subject.O = desc subject.OU = 'Created at ' + time.strftime('%F %T%z') if common is not None: subject.CN = common issuer_pkey = issuer_subject = issuer_fingerprint = serial = None if issuer is None: isCA = True else: isCA = False if isinstance(issuer, basestring): # Look up CA by purpose issuer_x509, issuer_pkey = self.getCertificatePair(issuer) else: # Tuple provided issuer_x509, issuer_pkey = issuer issuer_x509 = X509.load_cert_string(issuer_x509) issuer_pkey = EVP.load_key_string(issuer_pkey) issuer_subject = issuer_x509.get_subject() issuer_fingerprint = digestlib.sha1( issuer_x509.as_der()).hexdigest() cu.execute("""UPDATE pki_certificates SET ca_serial_index = ca_serial_index + 1 WHERE fingerprint = ? RETURNING ca_serial_index """, issuer_fingerprint) serial, = cu.fetchone() # Create certificates with a 'not before' date 1 day in the past, just # in case initial setup sets the clock backwards. rsa, x509 = gencert.new_cert(KEY_LENGTH, subject, EXPIRY, issuer=issuer_subject, issuer_evp=issuer_pkey, isCA=isCA, serial=serial, timestamp_offset=-86400) fingerprint = digestlib.sha1(x509.as_der()).hexdigest() pkey_pem = rsa.as_pem(None) x509_pem = x509.as_pem() cu.execute("""INSERT INTO pki_certificates ( fingerprint, purpose, is_ca, x509_pem, pkey_pem, issuer_fingerprint, ca_serial_index, time_issued, time_expired ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", fingerprint, purpose, isCA, x509_pem, pkey_pem, issuer_fingerprint, 0, str(x509.get_not_before()), str(x509.get_not_after()), ) log.info("Created certificate %s for purpose %r%s", fingerprint, purpose, (issuer_fingerprint and (" (issuer %s)" % issuer_fingerprint) or "")) return x509_pem, pkey_pem
def seek(self, where): # This allows the conary http client to rewind the body file, and # resets the digest along with it. assert where == 0 self.digest = digestlib.sha1() self.fobj.seek(0)