def _do_checksum(self):
"""return a checksum for a package:
- check if the checksum cache is enabled
if not - return the checksum
if so - check to see if it has a cache file
if so, open it and return the first line's contents
if not, grab the checksum and write it to a file for this pkg
"""
# already got it
if self._checksum:
return self._checksum
# not using the cachedir
if not self._cachedir:
self._checksum = misc.checksum(self.checksum_type, self.localpath)
self._checksums = [(self.checksum_type, self._checksum, 1)]
return self._checksum
t = []
if type(self.hdr[rpm.RPMTAG_SIGGPG]) is not types.NoneType:
t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))
if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType:
t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType:
t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
kcsum = misc.Checksums(checksums=[self.checksum_type])
kcsum.update("".join(t))
key = kcsum.hexdigest()
csumtag = '%s-%s-%s-%s' % (os.path.basename(self.localpath),
key, self.size, self.filetime)
csumfile = '%s/%s' % (self._cachedir, csumtag)
if os.path.exists(csumfile) and float(self.filetime) <= float(os.stat(csumfile)[-2]):
csumo = open(csumfile, 'r')
checksum = csumo.readline()
csumo.close()
else:
checksum = misc.checksum(self.checksum_type, self.localpath)
# This is atomic cache creation via. rename, so we can have two
# tasks using the same cachedir ... mash does this.
try:
(csumo, tmpfilename) = tempfile.mkstemp(dir=self._cachedir)
csumo = os.fdopen(csumo, 'w', -1)
csumo.write(checksum)
csumo.close()
os.rename(tmpfilename, csumfile)
except:
pass
self._checksum = checksum
self._checksums = [(self.checksum_type, checksum, 1)]
return self._checksum
def _do_checksum(self):
"""return a checksum for a package:
- check if the checksum cache is enabled
if not - return the checksum
if so - check to see if it has a cache file
if so, open it and return the first line's contents
if not, grab the checksum and write it to a file for this pkg
"""
# already got it
if self._checksum:
return self._checksum
# not using the cachedir
if not hasattr(self, '_cachedir') or not self._cachedir:
self._checksum = misc.checksum(self.checksum_type, self.localpath)
self._checksums = [(self.checksum_type, self._checksum, 1)]
return self._checksum
t = []
if type(self.hdr[rpm.RPMTAG_SIGGPG]) is not types.NoneType:
t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))
if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType:
t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType:
t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
kcsum = misc.Checksums(checksums=[self.checksum_type])
kcsum.update("".join(t))
key = kcsum.hexdigest()
csumtag = '%s-%s-%s-%s' % (os.path.basename(
self.localpath), key, self.size, self.filetime)
csumfile = '%s/%s' % (self._cachedir, csumtag)
if os.path.exists(csumfile) and float(self.filetime) <= float(
os.stat(csumfile)[-2]):
csumo = open(csumfile, 'r')
checksum = csumo.readline()
csumo.close()
else:
checksum = misc.checksum(self.checksum_type, self.localpath)
# This is atomic cache creation via. rename, so we can have two
# tasks using the same cachedir ... mash does this.
try:
(csumo, tmpfilename) = tempfile.mkstemp(dir=self._cachedir)
csumo = os.fdopen(csumo, 'w', -1)
csumo.write(checksum)
csumo.close()
os.rename(tmpfilename, csumfile)
except:
pass
self._checksum = checksum
self._checksums = [(self.checksum_type, checksum, 1)]
return self._checksum
class DeltaRPMPackage:
"""each drpm is one object, you pass it a drpm file
it opens the file, and pulls the information out in bite-sized chunks :)
"""
mode_cache = {}
def __init__(self, po, basedir, filename):
try:
stats = os.stat(os.path.join(basedir, filename))
self.size = stats[6]
self.mtime = stats[8]
del stats
except OSError, e:
raise MDError, "Error Stat'ing file %s%s" % (basedir, filename)
self.csum_type = 'sha256'
self.relativepath = filename
self.po = po
fd = os.open(self.po.localpath, os.O_RDONLY)
os.lseek(fd, 0, 0)
fo = os.fdopen(fd, 'rb')
self.csum = misc.checksum(self.csum_type, fo)
del fo
del fd
self._getDRPMInfo(os.path.join(basedir, filename))
def checksum_and_rename(fn_path, sumtype='sha256'):
"""checksum the file rename the file to contain the checksum as a prefix
return the new filename"""
csum = misc.checksum(sumtype, fn_path)
fn = os.path.basename(fn_path)
fndir = os.path.dirname(fn_path)
csum_fn = csum + '-' + fn
csum_path = os.path.join(fndir, csum_fn)
os.rename(fn_path, csum_path)
return (csum, csum_path)
def checksum_and_rename(fn_path, sumtype='sha256'):
"""checksum the file rename the file to contain the checksum as a prefix
return the new filename"""
csum = misc.checksum(sumtype, fn_path)
fn = os.path.basename(fn_path)
fndir = os.path.dirname(fn_path)
csum_fn = csum + '-' + fn
csum_path = os.path.join(fndir, csum_fn)
os.rename(fn_path, csum_path)
return (csum, csum_path)
def checksum_and_rename(fn_path, sumtype='sha256'):
"""checksum the file rename the file to contain the checksum as a prefix
return the new filename"""
csum = misc.checksum(sumtype, fn_path)
fn = os.path.basename(fn_path)
fndir = os.path.dirname(fn_path)
fn_match = re.match(r'[0-9A-Fa-f]{32,128}-(.+)', fn)
if fn_match:
fn = fn_match.groups()[0]
csum_fn = csum + '-' + fn
csum_path = os.path.join(fndir, csum_fn)
os.rename(fn_path, csum_path)
return (csum, csum_path)
def checksum_and_rename(fn_path, sumtype='sha256'):
"""checksum the file rename the file to contain the checksum as a prefix
return the new filename"""
csum = misc.checksum(sumtype, fn_path)
fn = os.path.basename(fn_path)
fndir = os.path.dirname(fn_path)
fn_match = re.match(r'[0-9A-Fa-f]{32,128}-(.+)', fn)
if fn_match:
fn = fn_match.groups()[0]
csum_fn = csum + '-' + fn
csum_path = os.path.join(fndir, csum_fn)
os.rename(fn_path, csum_path)
return (csum, csum_path)
def checkfileurl(pkg):
pkg_path = pkg.remote_url
pkg_path = pkg_path.replace('file://', '')
(csum_type, csum) = pkg.returnIdSum()
try:
filesum = checksum(csum_type, pkg_path)
except Errors.MiscError:
return False
if filesum != csum:
return False
return True
def checkfileurl(pkg):
pkg_path = pkg.remote_url
pkg_path = pkg_path.replace('file://', '')
(csum_type, csum) = pkg.returnIdSum()
try:
filesum = checksum(csum_type, pkg_path)
except Errors.MiscError:
return False
if filesum != csum:
return False
return True
def verifyLocalPkg(self):
# check file size first
try: fsize = os.path.getsize(self.localpath)
except OSError: return False
if fsize != self.size: return False
# checksum
ctype, csum = self.csum
try: fsum = checksum(ctype, self.localpath)
except MiscError: return False
if fsum != csum: return False
# hooray
return True
def verifyLocalPkg(self):
# check file size first
try:
fsize = os.path.getsize(self.localpath)
except OSError:
return False
if fsize != self.size: return False
# checksum
ctype, csum = self.csum
try:
fsum = checksum(ctype, self.localpath)
except MiscError:
return False
if fsum != csum: return False
# hooray
return True
def sanity_check_repodata(myurl):
"""
Sanity check the repodata for a given repository.
Initial implementation by Seth Vidal.
"""
myurl = str(myurl)
tempdir = tempfile.mkdtemp()
errorstrings = []
if myurl[-1] != '/':
myurl += '/'
baseurl = myurl
if not myurl.endswith('repodata/'):
myurl += 'repodata/'
else:
baseurl = baseurl.replace('repodata/', '/')
rf = myurl + 'repomd.xml'
try:
rm = urlgrabber.urlopen(rf)
repomd = repoMDObject.RepoMD('foo', rm)
for t in repomd.fileTypes():
data = repomd.getData(t)
base, href = data.location
if base:
loc = base + '/' + href
else:
loc = baseurl + href
destfn = tempdir + '/' + os.path.basename(href)
dest = urlgrabber.urlgrab(loc, destfn)
ctype, known_csum = data.checksum
csum = checksum(ctype, dest)
if csum != known_csum:
errorstrings.append("checksum: %s" % t)
if href.find('xml') != -1:
decompressed = decompress(dest)
retcode = subprocess.call(
['/usr/bin/xmllint', '--noout', decompressed])
if retcode != 0:
errorstrings.append("failed xml read: %s" % t)
except urlgrabber.grabber.URLGrabError, e:
errorstrings.append('Error accessing repository %s' % e)
def treeinfo_checksum(treeinfo):
# read treeinfo file into cp
# take checksums section
result = 0
cp = LocalConfigParser()
try:
cp.read(treeinfo)
except ConfigParser.MissingSectionHeaderError:
# Generally this means we failed to access the file
print " could not find sections in treeinfo file %s" % treeinfo
return BAD_IMAGES
except ConfigParser.Error:
print " could not parse treeinfo file %s" % treeinfo
return BAD_IMAGES
if not cp.has_section('checksums'):
print " no checksums section in treeinfo file %s" % treeinfo
return BAD_IMAGES
dir_path = os.path.dirname(treeinfo)
for opt in cp.options('checksums'):
fnpath = dir_path + '/%s' % opt
fnpath = os.path.normpath(fnpath)
csuminfo = cp.get('checksums', opt).split(':')
if len(csuminfo) < 2:
print " checksum information doesn't make any sense for %s." % opt
result = BAD_IMAGES
continue
if not os.path.exists(fnpath):
print " cannot find file %s listed in treeinfo" % fnpath
result = BAD_IMAGES
continue
csum = checksum(csuminfo[0], fnpath)
if csum != csuminfo[1]:
print " file %s %s does not match:\n ondisk %s vs treeinfo: %s" % (
opt, csuminfo[0], csum, csuminfo[1])
result = BAD_IMAGES
continue
return result
def sanity_check_repodata(myurl):
"""
Sanity check the repodata for a given repository.
Initial implementation by Seth Vidal.
"""
myurl = str(myurl)
tempdir = tempfile.mkdtemp()
errorstrings = []
if myurl[-1] != '/':
myurl += '/'
baseurl = myurl
if not myurl.endswith('repodata/'):
myurl += 'repodata/'
else:
baseurl = baseurl.replace('repodata/', '/')
rf = myurl + 'repomd.xml'
try:
rm = urlgrabber.urlopen(rf)
repomd = repoMDObject.RepoMD('foo', rm)
for t in repomd.fileTypes():
data = repomd.getData(t)
base, href = data.location
if base:
loc = base + '/' + href
else:
loc = baseurl + href
destfn = tempdir + '/' + os.path.basename(href)
dest = urlgrabber.urlgrab(loc, destfn)
ctype, known_csum = data.checksum
csum = checksum(ctype, dest)
if csum != known_csum:
errorstrings.append("checksum: %s" % t)
if href.find('xml') != -1:
decompressed = decompress(dest)
retcode = subprocess.call(['/usr/bin/xmllint', '--noout', decompressed])
if retcode != 0:
errorstrings.append("failed xml read: %s" % t)
except urlgrabber.grabber.URLGrabError, e:
errorstrings.append('Error accessing repository %s' % e)
def treeinfo_checksum(treeinfo):
# read treeinfo file into cp
# take checksums section
result = 0
cp = LocalConfigParser()
try:
cp.read(treeinfo)
except ConfigParser.MissingSectionHeaderError:
# Generally this means we failed to access the file
print " could not find sections in treeinfo file %s" % treeinfo
return BAD_IMAGES
except ConfigParser.Error:
print " could not parse treeinfo file %s" % treeinfo
return BAD_IMAGES
if not cp.has_section('checksums'):
print " no checksums section in treeinfo file %s" % treeinfo
return BAD_IMAGES
dir_path = os.path.dirname(treeinfo)
for opt in cp.options('checksums'):
fnpath = dir_path + '/%s' % opt
fnpath = os.path.normpath(fnpath)
csuminfo = cp.get('checksums', opt).split(':')
if len(csuminfo) < 2:
print " checksum information doesn't make any sense for %s." % opt
result = BAD_IMAGES
continue
if not os.path.exists(fnpath):
print " cannot find file %s listed in treeinfo" % fnpath
result = BAD_IMAGES
continue
csum = checksum(csuminfo[0], fnpath)
if csum != csuminfo[1]:
print " file %s %s does not match:\n ondisk %s vs treeinfo: %s" % (opt, csuminfo[0], csum, csuminfo[1])
result = BAD_IMAGES
continue
return result
def add(self, metadata, mdtype=None):
""" Insert arbitrary metadata into this repository.
metadata can be either an xml.dom.minidom.Document object, or
a filename.
"""
md = None
if not metadata:
raise MDError, 'metadata cannot be None'
if isinstance(metadata, minidom.Document):
md = metadata.toxml()
mdname = 'updateinfo.xml'
oldmd = AutoFileChecksums(StringIO(md), [self.checksum_type])
oldmd.read()
elif isinstance(metadata, str):
if os.path.exists(metadata):
mdname = os.path.basename(metadata)
if mdname.split('.')[-1] in ('gz', 'bz2', 'xz'):
mdname = mdname.rsplit('.', 1)[0]
oldmd = compressOpen(metadata, mode='rb')
else:
oldmd = file(metadata, 'r')
oldmd = AutoFileChecksums(oldmd, [self.checksum_type])
md = oldmd.read()
oldmd.close()
else:
raise MDError, '%s not found' % metadata
else:
raise MDError, 'invalid metadata type'
if not md and self.compress_type == 'xz':
raise MDError, 'LZMA does not support compressing empty files'
## Compress the metadata and move it into the repodata
mdtype = self._get_mdtype(mdname, mdtype)
destmd = os.path.join(self.repodir, mdname)
if self.compress:
destmd += '.' + self.compress_type
newmd = compressOpen(destmd,
mode='wb',
compress_type=self.compress_type)
else:
newmd = open(destmd, 'wb')
newmd.write(md)
newmd.close()
print "Wrote:", destmd
if self.unique_md_filenames:
csum, destmd = checksum_and_rename(destmd, self.checksum_type)
else:
csum = checksum(self.checksum_type, destmd)
base_destmd = os.path.basename(destmd)
# Remove any stale metadata
old_rd = self.repoobj.repoData.pop(mdtype, None)
new_rd = RepoData()
new_rd.type = mdtype
new_rd.location = (None, 'repodata/' + base_destmd)
new_rd.checksum = (self.checksum_type, csum)
new_rd.size = str(os.stat(destmd).st_size)
if self.compress:
new_rd.openchecksum = (self.checksum_type,
oldmd.checksums.hexdigests().popitem()[1])
new_rd.opensize = str(oldmd.checksums.length)
new_rd.timestamp = str(int(os.stat(destmd).st_mtime))
self.repoobj.repoData[new_rd.type] = new_rd
self._print_repodata(new_rd)
self._write_repomd()
if old_rd is not None and old_rd.location[1] != new_rd.location[1]:
# remove the old file when overwriting metadata
# with the same mdtype but different location
self._remove_repodata_file(old_rd)
def add(self, metadata, mdtype=None):
""" Insert arbitrary metadata into this repository.
metadata can be either an xml.dom.minidom.Document object, or
a filename.
"""
md = None
if not metadata:
raise MDError, 'metadata cannot be None'
if isinstance(metadata, minidom.Document):
md = metadata.toxml()
mdname = 'updateinfo.xml'
elif isinstance(metadata, str):
if os.path.exists(metadata):
mdname = os.path.basename(metadata)
if mdname.split('.')[-1] in ('gz', 'bz2', 'xz'):
mdname = mdname.rsplit('.', 1)[0]
oldmd = compressOpen(metadata, mode='rb')
else:
oldmd = file(metadata, 'r')
oldmd = AutoFileChecksums(oldmd, [self.checksum_type])
md = oldmd.read()
oldmd.close()
else:
raise MDError, '%s not found' % metadata
else:
raise MDError, 'invalid metadata type'
## Compress the metadata and move it into the repodata
mdtype = self._get_mdtype(mdname, mdtype)
destmd = os.path.join(self.repodir, mdname)
if self.compress:
destmd += '.' + self.compress_type
newmd = compressOpen(destmd, mode='wb', compress_type=self.compress_type)
else:
newmd = open(destmd, 'wb')
newmd.write(md)
newmd.close()
print "Wrote:", destmd
if self.unique_md_filenames:
csum, destmd = checksum_and_rename(destmd, self.checksum_type)
else:
csum = checksum(self.checksum_type, destmd)
base_destmd = os.path.basename(destmd)
# Remove any stale metadata
old_rd = self.repoobj.repoData.pop(mdtype, None)
new_rd = RepoData()
new_rd.type = mdtype
new_rd.location = (None, 'repodata/' + base_destmd)
new_rd.checksum = (self.checksum_type, csum)
new_rd.size = str(os.stat(destmd).st_size)
if self.compress:
new_rd.openchecksum = oldmd.checksums.hexdigests().popitem()
new_rd.opensize = str(oldmd.checksums.length)
new_rd.timestamp = str(int(os.stat(destmd).st_mtime))
self.repoobj.repoData[new_rd.type] = new_rd
self._print_repodata(new_rd)
self._write_repomd()
if old_rd is not None and old_rd.location[1] != new_rd.location[1]:
# remove the old file when overwriting metadata
# with the same mdtype but different location
self._remove_repodata_file(old_rd)
def add(self, metadata, mdtype=None):
""" Insert arbitrary metadata into this repository.
metadata can be either an xml.dom.minidom.Document object, or
a filename.
"""
md = None
if not metadata:
raise MDError, 'metadata cannot be None'
if isinstance(metadata, minidom.Document):
md = metadata.toxml()
mdname = 'updateinfo.xml'
elif isinstance(metadata, str):
if os.path.exists(metadata):
if metadata.split('.')[-1] in ('gz', 'bz2', 'xz'):
oldmd = compressOpen(metadata, mode='rb')
else:
oldmd = file(metadata, 'r')
md = oldmd.read()
oldmd.close()
mdname = os.path.basename(metadata)
else:
raise MDError, '%s not found' % metadata
else:
raise MDError, 'invalid metadata type'
do_compress = False
## Compress the metadata and move it into the repodata
if self.compress or not mdname.split('.')[-1] in ('gz', 'bz2', 'xz'):
do_compress = True
mdname += '.' + self.compress_type
mdtype = self._get_mdtype(mdname, mdtype)
destmd = os.path.join(self.repodir, mdname)
if do_compress:
newmd = compressOpen(destmd,
mode='wb',
compress_type=self.compress_type)
else:
newmd = open(destmd, 'wb')
newmd.write(md)
newmd.close()
print "Wrote:", destmd
open_csum = checksum(self.checksum_type, metadata)
csum, destmd = checksum_and_rename(destmd, self.checksum_type)
base_destmd = os.path.basename(destmd)
# Remove any stale metadata
old_rd = self.repoobj.repoData.pop(mdtype, None)
new_rd = RepoData()
new_rd.type = mdtype
new_rd.location = (None, 'repodata/' + base_destmd)
new_rd.checksum = (self.checksum_type, csum)
new_rd.openchecksum = (self.checksum_type, open_csum)
new_rd.size = str(os.stat(destmd).st_size)
new_rd.timestamp = str(os.stat(destmd).st_mtime)
self.repoobj.repoData[new_rd.type] = new_rd
self._print_repodata(new_rd)
self._write_repomd()
if old_rd is not None and old_rd.location[1] != new_rd.location[1]:
# remove the old file when overwriting metadata
# with the same mdtype but different location
self._remove_repodata_file(old_rd)
def add(self, metadata, mdtype=None):
""" Insert arbitrary metadata into this repository.
metadata can be either an xml.dom.minidom.Document object, or
a filename.
"""
md = None
if not metadata:
raise MDError, 'metadata cannot be None'
if isinstance(metadata, minidom.Document):
md = metadata.toxml()
mdname = 'updateinfo.xml'
elif isinstance(metadata, str):
if os.path.exists(metadata):
if metadata.endswith('.gz'):
oldmd = GzipFile(filename=metadata, mode='rb')
else:
oldmd = file(metadata, 'r')
md = oldmd.read()
oldmd.close()
mdname = os.path.basename(metadata)
else:
raise MDError, '%s not found' % metadata
else:
raise MDError, 'invalid metadata type'
## Compress the metadata and move it into the repodata
if not mdname.endswith('.gz'):
mdname += '.gz'
if not mdtype:
mdtype = mdname.split('.')[0]
destmd = os.path.join(self.repodir, mdname)
newmd = GzipFile(filename=destmd, mode='wb')
newmd.write(md)
newmd.close()
print "Wrote:", destmd
open_csum = checksum(self.checksum_type, metadata)
csum, destmd = checksum_and_rename(destmd, self.checksum_type)
base_destmd = os.path.basename(destmd)
## Remove any stale metadata
if mdtype in self.repoobj.repoData:
del self.repoobj.repoData[mdtype]
new_rd = RepoData()
new_rd.type = mdtype
new_rd.location = (None, 'repodata/' + base_destmd)
new_rd.checksum = (self.checksum_type, csum)
new_rd.openchecksum = (self.checksum_type, open_csum)
new_rd.size = str(os.stat(destmd).st_size)
new_rd.timestamp = str(os.stat(destmd).st_mtime)
self.repoobj.repoData[new_rd.type] = new_rd
print " type =", new_rd.type
print " location =", new_rd.location[1]
print " checksum =", new_rd.checksum[1]
print " timestamp =", new_rd.timestamp
print " open-checksum =", new_rd.openchecksum[1]
## Write the updated repomd.xml
outmd = file(self.repomdxml, 'w')
outmd.write(self.repoobj.dump_xml())
outmd.close()
print "Wrote:", self.repomdxml
