class TestPkgCache(unittest.TestCase): def setUp(self): """Initialize the environment.""" self.cache = PkgCache('/tmp/cache', force_clean=True) for fn in ('file_a', 'file_b', 'file_c'): with open(os.path.join('/tmp', fn), 'w') as f: print >>f, fn def tearDown(self): """Clean the environment.""" shutil.rmtree('/tmp/cache') for fn in ('/tmp/file_a', '/tmp/file_b', '/tmp/file_c'): os.unlink(fn) def test_insertion(self): self.cache[('file_a', 1)] = '/tmp/file_a' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertEqual(open('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a').read(), 'file_a\n') self.cache[('file_b', 1)] = '/tmp/file_b' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) self.assertEqual(open('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3').read(), 'file_b\n') self.cache[('file_c', 1)] = '/tmp/file_c' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) self.assertEqual(open('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f').read(), 'file_c\n') def test_index(self): self.cache[('file_a', 1)] = '/tmp/file_a' self.assertEqual(self.cache[('file_a', 1)], ('c7f33375edf32d8fb62d4b505c74519a', 'file_a')) self.cache[('file_b', 1)] = '/tmp/file_b' self.assertEqual(self.cache[('file_b', 1)], ('a7004efbb89078ebcc8f21d55354e2f3', 'file_b')) self.cache[('file_c', 1)] = '/tmp/file_c' self.assertEqual(self.cache[('file_c', 1)], ('22ee05516c08f3672cb25e03ce7f045f', 'file_c')) self.assertEqual(set(self.cache.keys()), set((('file_a', 1), ('file_b', 1), ('file_c', 1)))) def test_delete(self): self.cache[('file_a', 1)] = '/tmp/file_a' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) del self.cache[('file_a', 1)] self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) self.cache[('file_b', 1)] = '/tmp/file_b' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) del self.cache[('file_b', 1)] self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7/004efbb89078ebcc8f21d55354e2f3')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/a7')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) self.cache[('file_c', 1)] = '/tmp/file_c' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) del self.cache[('file_c', 1)] self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22/ee05516c08f3672cb25e03ce7f045f')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/22')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache')) def test_collision(self): self.cache[('file_a', 1)] = '/tmp/file_a' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.cache[('file_a', 2)] = '/tmp/file_a' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) self.cache[('file_a', 3)] = '/tmp/file_a' self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) del self.cache[('file_a', 2)] self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) del self.cache[('file_a', 1)] self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) del self.cache[('file_a', 3)] self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-001')) self.assertFalse(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a-002')) def test_linkto(self): self.cache[('file_a', 1)] = '/tmp/file_a' self.cache.linkto(('file_a', 1), '/tmp/file_a_') self.assertEqual(open('/tmp/file_a_').read(), 'file_a\n') os.unlink('/tmp/file_a_') self.assertTrue(os.path.exists('/tmp/cache/pkgcache/c7/f33375edf32d8fb62d4b505c74519a')) def test_clean(self): self.cache[('file_a', 1)] = '/tmp/file_a' self.cache[('file_a', 2)] = '/tmp/file_a' self.cache[('file_a', 3)] = '/tmp/file_a' self.cache[('file_b', 1)] = '/tmp/file_b' self.cache[('file_c', 1)] = '/tmp/file_c' osclib.pkgcache.time.time = MagicMock(return_value=3) self.cache._clean_cache(ttl=2) self.assertFalse(('file_a', 1) in self.cache) self.assertFalse(('file_a', 2) in self.cache) self.assertTrue(('file_a', 3) in self.cache) self.assertFalse(('file_b', 1) in self.cache) self.assertFalse(('file_c', 1) in self.cache)
class ABIChecker(ReviewBot.ReviewBot): """ check ABI of library packages """ def __init__(self, *args, **kwargs): ReviewBot.ReviewBot.__init__(self, *args, **kwargs) self.no_review = False self.force = False self.ts = rpm.TransactionSet() self.ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) self.pkgcache = PkgCache(BINCACHE) # reports of source submission self.reports = [] # textual report summary for use in accept/decline message # or comments self.text_summary = '' self.session = DB.db_session() self.dblogger = LogToDB(self.session) self.logger.addFilter(self.dblogger) self.commentapi = CommentAPI(self.apiurl) def check_source_submission(self, src_project, src_package, src_rev, dst_project, dst_package): # happens for maintenance incidents if dst_project == None and src_package == 'patchinfo': return None if dst_project in PROJECT_BLACKLIST: self.logger.info(PROJECT_BLACKLIST[dst_project]) # self.text_summary += PROJECT_BLACKLIST[dst_project] + "\n" return True # default is to accept the review, just leave a note if # there were problems. ret = True ReviewBot.ReviewBot.check_source_submission(self, src_project, src_package, src_rev, dst_project, dst_package) report = Report(src_project, src_package, src_rev, dst_project, dst_package, [], None) dst_srcinfo = self.get_sourceinfo(dst_project, dst_package) self.logger.debug('dest sourceinfo %s', pformat(dst_srcinfo)) if dst_srcinfo is None: msg = "%s/%s seems to be a new package, no need to review" % ( dst_project, dst_package) self.logger.info(msg) self.text_summary += msg + "\n" self.reports.append(report) return True src_srcinfo = self.get_sourceinfo(src_project, src_package, src_rev) self.logger.debug('src sourceinfo %s', pformat(src_srcinfo)) if src_srcinfo is None: msg = "%s/%s@%s does not exist!? can't check" % ( src_project, src_package, src_rev) self.logger.error(msg) self.text_summary += msg + "\n" self.reports.append(report) return False if os.path.exists(UNPACKDIR): shutil.rmtree(UNPACKDIR) try: # compute list of common repos to find out what to compare myrepos = self.findrepos(src_project, src_srcinfo, dst_project, dst_srcinfo) except NoBuildSuccess as e: self.logger.info(e) self.text_summary += "**Error**: %s\n" % e self.reports.append(report) return False except NotReadyYet as e: self.logger.info(e) self.reports.append(report) return None except SourceBroken as e: self.logger.error(e) self.text_summary += "**Error**: %s\n" % e self.reports.append(report) return False if not myrepos: self.text_summary += "**Error**: %s does not build against %s, can't check library ABIs\n\n" % ( src_project, dst_project) self.logger.info("no matching repos, can't compare") self.reports.append(report) return False # *** beware of nasty maintenance stuff *** # if the destination is a maintained project we need to # mangle our comparison target and the repo mapping try: originproject, originpackage, origin_srcinfo, new_repo_map = self._maintenance_hack( dst_project, dst_srcinfo, myrepos) if originproject is not None: dst_project = originproject if originpackage is not None: dst_package = originpackage if origin_srcinfo is not None: dst_srcinfo = origin_srcinfo if new_repo_map is not None: myrepos = new_repo_map except MaintenanceError as e: self.text_summary += "**Error**: %s\n\n" % e self.logger.error('%s', e) self.reports.append(report) return False except NoBuildSuccess as e: self.logger.info(e) self.text_summary += "**Error**: %s\n" % e self.reports.append(report) return False except NotReadyYet as e: self.logger.info(e) self.reports.append(report) return None except SourceBroken as e: self.logger.error(e) self.text_summary += "**Error**: %s\n" % e self.reports.append(report) return False notes = [] libresults = [] overall = None missing_debuginfo = [] for mr in myrepos: try: dst_libs = self.extract(dst_project, dst_package, dst_srcinfo, mr.dstrepo, mr.arch) # nothing to fetch, so no libs if dst_libs is None: continue except DistUrlMismatch as e: self.logger.error( "%s/%s %s/%s: %s" % (dst_project, dst_package, mr.dstrepo, mr.arch, e)) if ret == True: # need to check again ret = None continue except MissingDebugInfo as e: missing_debuginfo.append(str(e)) ret = False continue except FetchError as e: self.logger.error(e) if ret == True: # need to check again ret = None continue try: src_libs = self.extract(src_project, src_package, src_srcinfo, mr.srcrepo, mr.arch) if src_libs is None: if dst_libs: self.text_summary += "*Warning*: the submission does not contain any libs anymore\n\n" continue except DistUrlMismatch as e: self.logger.error( "%s/%s %s/%s: %s" % (src_project, src_package, mr.srcrepo, mr.arch, e)) if ret == True: # need to check again ret = None continue except MissingDebugInfo as e: missing_debuginfo.append(str(e)) ret = False continue except FetchError as e: self.logger.error(e) if ret == True: # need to check again ret = None continue # create reverse index for aliases in the source project src_aliases = dict() for lib in src_libs.keys(): for a in src_libs[lib]: src_aliases.setdefault(a, set()).add(lib) # for each library in the destination project check if the same lib # exists in the source project. If not check the aliases (symlinks) # to catch soname changes. Generate pairs of matching libraries. pairs = set() for lib in dst_libs.keys(): if lib in src_libs: pairs.add((lib, lib)) else: self.logger.debug( "%s not found in submission, checking aliases", lib) found = False for a in dst_libs[lib]: if a in src_aliases: for l in src_aliases[a]: pairs.add((lib, l)) found = True if found == False: self.text_summary += "*Warning*: %s no longer packaged\n\n" % lib self.logger.debug("to diff: %s", pformat(pairs)) # for each pair dump and compare the abi for old, new in pairs: # abi dump of old lib new_base = os.path.join(UNPACKDIR, dst_project, dst_package, mr.dstrepo, mr.arch) old_dump = os.path.join(CACHEDIR, 'old.dump') # abi dump of new lib old_base = os.path.join(UNPACKDIR, src_project, src_package, mr.srcrepo, mr.arch) new_dump = os.path.join(CACHEDIR, 'new.dump') def cleanup(): if os.path.exists(old_dump): os.unlink(old_dump) if os.path.exists(new_dump): os.unlink(new_dump) cleanup() # we just need that to pass a name to abi checker m = so_re.match(old) htmlreport = 'report-%s-%s-%s-%s-%s-%08x.html' % ( mr.srcrepo, os.path.basename(old), mr.dstrepo, os.path.basename(new), mr.arch, time.time()) # run abichecker if m \ and self.run_abi_dumper(old_dump, new_base, old) \ and self.run_abi_dumper(new_dump, old_base, new): reportfn = os.path.join(CACHEDIR, htmlreport) r = self.run_abi_checker(m.group(1), old_dump, new_dump, reportfn) if r is not None: self.logger.debug('report saved to %s, compatible: %d', reportfn, r) libresults.append( LibResult(mr.srcrepo, os.path.basename(old), mr.dstrepo, os.path.basename(new), mr.arch, htmlreport, r)) if overall is None: overall = r elif overall == True and r == False: overall = r else: self.logger.error('failed to compare %s <> %s' % (old, new)) self.text_summary += "**Error**: ABI check failed on %s vs %s\n\n" % ( old, new) if ret == True: # need to check again ret = None cleanup() if missing_debuginfo: self.text_summary += 'debug information is missing for the following packages, can\'t check:\n<pre>' self.text_summary += ''.join(missing_debuginfo) self.text_summary += '</pre>\nplease enable debug info in your project config.\n' self.reports.append(report._replace(result=overall, reports=libresults)) # upload reports if os.path.exists(UNPACKDIR): shutil.rmtree(UNPACKDIR) return ret def _maintenance_hack(self, dst_project, dst_srcinfo, myrepos): pkg = dst_srcinfo.package originproject = None originpackage = None # find the maintenance project url = osc.core.makeurl( self.apiurl, ('search', 'project', 'id'), "match=(maintenance/maintains/@project='%s'+and+attribute/@name='%s')" % (dst_project, osc.conf.config['maintenance_attribute'])) root = ET.parse(osc.core.http_GET(url)).getroot() if root is not None: node = root.find('project') if node is not None: # check if target project is a project link where the # sources don't actually build (like openSUSE:...:Update). That # is the case if no update was released yet. # XXX: TODO: do check for whether the package builds here first originproject = self.get_originproject(dst_project, pkg) if originproject is not None: self.logger.debug("origin project %s", originproject) url = osc.core.makeurl(self.apiurl, ('build', dst_project, '_result'), {'package': pkg}) root = ET.parse(osc.core.http_GET(url)).getroot() alldisabled = True for node in root.findall('status'): if node.get('code') != 'disabled': alldisabled = False if alldisabled: self.logger.debug( "all repos disabled, using originproject %s" % originproject) else: originproject = None else: mproject = node.attrib['name'] # packages are only a link to packagename.incidentnr (linkprj, linkpkg) = self._get_linktarget(dst_project, pkg) if linkpkg is not None and linkprj == dst_project: self.logger.debug("%s/%s links to %s" % (dst_project, pkg, linkpkg)) regex = re.compile(r'.*\.(\d+)$') m = regex.match(linkpkg) if m is None: raise MaintenanceError( "%s/%s -> %s/%s is not a proper maintenance link (must match /%s/)" % (dst_project, pkg, linkprj, linkpkg, regex.pattern)) incident = m.group(1) self.logger.debug("is maintenance incident %s" % incident) originproject = "%s:%s" % (mproject, incident) originpackage = pkg + '.' + dst_project.replace( ':', '_') origin_srcinfo = self.get_sourceinfo( originproject, originpackage) if origin_srcinfo is None: raise MaintenanceError( "%s/%s invalid" % (originproject, originpackage)) # find the map of maintenance incident repos to destination repos originrepos = self.findrepos(originproject, origin_srcinfo, dst_project, dst_srcinfo) mapped = dict() for mr in originrepos: mapped[(mr.dstrepo, mr.arch)] = mr self.logger.debug("mapping: %s", pformat(mapped)) # map the repos of the original request to the maintenance incident repos matchrepos = set() for mr in myrepos: if not (mr.dstrepo, mr.arch) in mapped: # sometimes a previously released maintenance # update didn't cover all architectures. We can # only ignore that then. self.logger.warn( "couldn't find repo %s/%s in %s/%s" % (mr.dstrepo, mr.arch, originproject, originpackage)) continue matchrepos.add( MR(mr.srcrepo, mapped[(mr.dstrepo, mr.arch)].srcrepo, mr.arch)) myrepos = matchrepos dst_srcinfo = origin_srcinfo self.logger.debug("new repo map: %s", pformat(myrepos)) return (originproject, originpackage, dst_srcinfo, myrepos) def find_abichecker_comment(self, req): """Return previous comments (should be one).""" comments = self.commentapi.get_comments(request_id=req.reqid) for c in comments.values(): m = comment_marker_re.match(c['comment']) if m: return c['id'], m.group('state'), m.group('result') return None, None, None def check_one_request(self, req): self.review_messages = ReviewBot.ReviewBot.DEFAULT_REVIEW_MESSAGES if self.no_review and not self.force and self.check_request_already_done( req.reqid): self.logger.info("skip request %s which is already done", req.reqid) # TODO: check if the request was seen before and we # didn't reach a final state for too long return None commentid, state, result = self.find_abichecker_comment(req) ## using comments instead of db would be an options for bots ## that use no db # if self.no_review: # if state == 'done': # self.logger.debug("request %s already done, result: %s"%(req.reqid, result)) # return self.dblogger.request_id = req.reqid self.reports = [] self.text_summary = '' try: ret = ReviewBot.ReviewBot.check_one_request(self, req) except Exception as e: import traceback self.logger.error("unhandled exception in ABI checker") self.logger.error(traceback.format_exc()) ret = None result = None if ret is not None: state = 'done' result = 'accepted' if ret else 'declined' else: # we probably don't want abichecker to spam here # FIXME don't delete comment in this case #if state is None and not self.text_summary: # self.text_summary = 'abichecker will take a look later' state = 'seen' self.save_reports_to_db(req, state, result) if ret is not None and self.text_summary == '': # if for some reason save_reports_to_db didn't produce a # summary we add one self.text_summary = "ABI checker result: [%s](%s/request/%s)" % ( result, WEB_URL, req.reqid) if commentid and not self.dryrun: self.commentapi.delete(commentid) self.post_comment(req, state, result) self.review_messages = { 'accepted': self.text_summary, 'declined': self.text_summary } if self.no_review: ret = None self.dblogger.request_id = None return ret def check_request_already_done(self, reqid): try: request = self.session.query( DB.Request).filter(DB.Request.id == reqid).one() if request.state == 'done': return True except sqlalchemy.orm.exc.NoResultFound as e: pass return False def save_reports_to_db(self, req, state, result): try: request = self.session.query( DB.Request).filter(DB.Request.id == req.reqid).one() for i in self.session.query(DB.ABICheck).filter( DB.ABICheck.request_id == request.id).all(): # yeah, we could be smarter here and update existing reports instead self.session.delete(i) self.session.flush() request.state = state request.result = result except sqlalchemy.orm.exc.NoResultFound as e: request = DB.Request( id=req.reqid, state=state, result=result, ) self.session.add(request) self.session.commit() for r in self.reports: abicheck = DB.ABICheck(request=request, src_project=r.src_project, src_package=r.src_package, src_rev=r.src_rev, dst_project=r.dst_project, dst_package=r.dst_package, result=r.result) self.session.add(abicheck) self.session.commit() if r.result is None: continue elif r.result: self.text_summary += "Good news from ABI check, " self.text_summary += "%s seems to be ABI [compatible](%s/request/%s):\n\n" % ( r.dst_package, WEB_URL, req.reqid) else: self.text_summary += "Warning: bad news from ABI check, " self.text_summary += "%s may be ABI [**INCOMPATIBLE**](%s/request/%s):\n\n" % ( r.dst_package, WEB_URL, req.reqid) for lr in r.reports: libreport = DB.LibReport( abicheck=abicheck, src_repo=lr.src_repo, src_lib=lr.src_lib, dst_repo=lr.dst_repo, dst_lib=lr.dst_lib, arch=lr.arch, htmlreport=lr.htmlreport, result=lr.result, ) self.session.add(libreport) self.session.commit() self.text_summary += "* %s (%s): [%s](%s/report/%d)\n" % ( lr.dst_lib, lr.arch, "compatible" if lr.result else "***INCOMPATIBLE***", WEB_URL, libreport.id) self.reports = [] def post_comment(self, req, state, result): if not self.text_summary: return msg = "<!-- abichecker state=%s%s -->\n" % (state, ' result=%s' % result if result else '') msg += self.text_summary self.logger.info("add comment: %s" % msg) if not self.dryrun: #self.commentapi.delete_from_where_user(self.review_user, request_id = req.reqid) self.commentapi.add_comment(request_id=req.reqid, comment=msg) def run_abi_checker(self, libname, old, new, output): cmd = [ 'abi-compliance-checker', '-lib', libname, '-old', old, '-new', new, '-report-path', output ] self.logger.debug(cmd) r = subprocess.Popen(cmd, close_fds=True, cwd=CACHEDIR).wait() if not r in (0, 1): self.logger.error('abi-compliance-checker failed') # XXX: record error return None return r == 0 def run_abi_dumper(self, output, base, filename): cmd = [ 'abi-dumper', '-o', output, '-lver', os.path.basename(filename), '/'.join([base, filename]) ] debuglib = '%s/usr/lib/debug/%s.debug' % (base, filename) if os.path.exists(debuglib): cmd.append(debuglib) self.logger.debug(cmd) r = subprocess.Popen(cmd, close_fds=True, cwd=CACHEDIR).wait() if r != 0: self.logger.error("failed to dump %s!" % filename) # XXX: record error return False return True def extract(self, project, package, srcinfo, repo, arch): # fetch cpio headers # check file lists for library packages fetchlist, liblist = self.compute_fetchlist(project, package, srcinfo, repo, arch) if not fetchlist: msg = "no libraries found in %s/%s %s/%s" % (project, package, repo, arch) self.logger.info(msg) return None # mtimes in cpio are not the original ones, so we need to fetch # that separately :-( mtimes = self._getmtimes(project, package, repo, arch) self.logger.debug("fetchlist %s", pformat(fetchlist)) self.logger.debug("liblist %s", pformat(liblist)) debugfiles = set(['/usr/lib/debug%s.debug' % f for f in liblist]) # fetch binary rpms downloaded = self.download_files(project, package, repo, arch, fetchlist, mtimes) # extract binary rpms tmpfile = os.path.join(CACHEDIR, "cpio") for fn in fetchlist: self.logger.debug("extract %s" % fn) with open(tmpfile, 'wb') as tmpfd: if not fn in downloaded: raise FetchError("%s was not downloaded!" % fn) self.logger.debug(downloaded[fn]) r = subprocess.call(['rpm2cpio', downloaded[fn]], stdout=tmpfd, close_fds=True) if r != 0: raise FetchError("failed to extract %s!" % fn) tmpfd.close() cpio = CpioRead(tmpfile) cpio.read() for ch in cpio: fn = ch.filename if fn.startswith('./'): # rpm payload is relative fn = fn[1:] self.logger.debug("cpio fn %s", fn) if not fn in liblist and not fn in debugfiles: continue dst = os.path.join(UNPACKDIR, project, package, repo, arch) dst += fn if not os.path.exists(os.path.dirname(dst)): os.makedirs(os.path.dirname(dst)) self.logger.debug("dst %s", dst) # the filehandle in the cpio archive is private so # open it again with open(tmpfile, 'rb') as cpiofh: cpiofh.seek(ch.dataoff, os.SEEK_SET) with open(dst, 'wb') as fh: while True: buf = cpiofh.read(4096) if buf is None or buf == '': break fh.write(buf) os.unlink(tmpfile) return liblist def download_files(self, project, package, repo, arch, filenames, mtimes): downloaded = dict() for fn in filenames: if not fn in mtimes: raise FetchError( "missing mtime information for %s, can't check" % fn) repodir = os.path.join(DOWNLOADS, package, project, repo) if not os.path.exists(repodir): os.makedirs(repodir) t = os.path.join(repodir, fn) self._get_binary_file(project, repo, arch, package, fn, t, mtimes[fn]) downloaded[fn] = t return downloaded # XXX: from repochecker def _get_binary_file(self, project, repository, arch, package, filename, target, mtime): """Get a binary file from OBS.""" # Check if the file is already there. key = (project, repository, arch, package, filename, mtime) if key in self.pkgcache: try: os.unlink(target) except: pass self.pkgcache.linkto(key, target) else: osc.core.get_binary_file(self.apiurl, project, repository, arch, filename, package=package, target_filename=target) self.pkgcache[key] = target def readRpmHeaderFD(self, fd): h = None try: h = self.ts.hdrFromFdno(fd) except rpm.error as e: if str(e) == "public key not available": print str(e) if str(e) == "public key not trusted": print str(e) if str(e) == "error reading package header": print str(e) h = None return h def _fetchcpioheaders(self, project, package, repo, arch): u = osc.core.makeurl(self.apiurl, ['build', project, repo, arch, package], ['view=cpioheaders']) try: r = osc.core.http_GET(u) except urllib2.HTTPError as e: raise FetchError('failed to fetch header information: %s' % e) tmpfile = NamedTemporaryFile(prefix="cpio-", delete=False) for chunk in r: tmpfile.write(chunk) tmpfile.close() cpio = CpioRead(tmpfile.name) cpio.read() rpm_re = re.compile('(.+\.rpm)-[0-9A-Fa-f]{32}$') for ch in cpio: # ignore errors if ch.filename == '.errors': continue # the filehandle in the cpio archive is private so # open it again with open(tmpfile.name, 'rb') as fh: fh.seek(ch.dataoff, os.SEEK_SET) h = self.readRpmHeaderFD(fh) if h is None: raise FetchError("failed to read rpm header for %s" % ch.filename) m = rpm_re.match(ch.filename) if m: yield m.group(1), h os.unlink(tmpfile.name) def _getmtimes(self, prj, pkg, repo, arch): """ returns a dict of filename: mtime """ url = osc.core.makeurl(self.apiurl, ('build', prj, repo, arch, pkg)) try: root = ET.parse(osc.core.http_GET(url)).getroot() except urllib2.HTTPError: return None return dict([(node.attrib['filename'], node.attrib['mtime']) for node in root.findall('binary')]) # modified from repochecker def _last_build_success(self, src_project, tgt_project, src_package, rev): """Return the last build success XML document from OBS.""" try: query = { 'lastsuccess': 1, 'package': src_package, 'pathproject': tgt_project, 'srcmd5': rev } url = osc.core.makeurl(self.apiurl, ('build', src_project, '_result'), query) return ET.parse(osc.core.http_GET(url)).getroot() except urllib2.HTTPError as e: if e.code != 404: self.logger.error('ERROR in URL %s [%s]' % (url, e)) raise pass return None def get_buildsuccess_repos(self, src_project, tgt_project, src_package, rev): root = self._last_build_success(src_project, tgt_project, src_package, rev) if root is None: return None # build list of repos as set of (name, arch) tuples repos = set() for repo in root.findall('repository'): name = repo.attrib['name'] for node in repo.findall('arch'): repos.add((name, node.attrib['arch'])) self.logger.debug("success repos: %s", pformat(repos)) return repos def get_dstrepos(self, project): url = osc.core.makeurl(self.apiurl, ('source', project, '_meta')) try: root = ET.parse(osc.core.http_GET(url)).getroot() except urllib2.HTTPError: return None repos = set() for repo in root.findall('repository'): name = repo.attrib['name'] if project in REPO_WHITELIST and name not in REPO_WHITELIST[ project]: continue for node in repo.findall('arch'): arch = node.text if project in ARCH_WHITELIST and arch not in ARCH_WHITELIST[ project]: continue repos.add((name, arch)) return repos def ensure_settled(self, src_project, src_srcinfo, matchrepos): """ make sure current build state is final so we're not tricked with half finished results""" rmap = dict() results = osc.core.get_package_results( self.apiurl, src_project, src_srcinfo.package, repository=[mr.srcrepo for mr in matchrepos], arch=[mr.arch for mr in matchrepos]) for result in results: for res in osc.core.result_xml_to_dicts(result): if not 'package' in res or res[ 'package'] != src_srcinfo.package: continue rmap[(res['repository'], res['arch'])] = res for mr in matchrepos: if not (mr.srcrepo, mr.arch) in rmap: self.logger.warn("%s/%s had no build success" % (mr.srcrepo, mr.arch)) raise NotReadyYet(src_project, src_srcinfo.package, "no result") if rmap[(mr.srcrepo, mr.arch)]['dirty']: self.logger.warn("%s/%s dirty" % (mr.srcrepo, mr.arch)) raise NotReadyYet(src_project, src_srcinfo.package, "dirty") code = rmap[(mr.srcrepo, mr.arch)]['code'] if code == 'broken': raise SourceBroken(src_project, src_srcinfo.package) if code != 'succeeded' and code != 'locked' and code != 'excluded': self.logger.warn("%s/%s not succeeded (%s)" % (mr.srcrepo, mr.arch, code)) raise NotReadyYet(src_project, src_srcinfo.package, code) def findrepos(self, src_project, src_srcinfo, dst_project, dst_srcinfo): # get target repos that had a successful build dstrepos = self.get_dstrepos(dst_project) if dstrepos is None: return None url = osc.core.makeurl(self.apiurl, ('source', src_project, '_meta')) try: root = ET.parse(osc.core.http_GET(url)).getroot() except urllib2.HTTPError: return None # set of source repo name, target repo name, arch matchrepos = set() for repo in root.findall('repository'): name = repo.attrib['name'] path = repo.findall('path') if path is None or len(path) != 1: self.logger.error("repo %s has more than one path" % name) continue prj = path[0].attrib['project'] if prj == 'openSUSE:Tumbleweed': prj = 'openSUSE:Factory' # XXX: hack if prj != dst_project: continue for node in repo.findall('arch'): arch = node.text dstname = path[0].attrib['repository'] if (dstname, arch) in dstrepos: matchrepos.add(MR(name, dstname, arch)) if not matchrepos: return None else: self.logger.debug('matched repos %s', pformat(matchrepos)) # make sure it's not dirty self.ensure_settled(src_project, src_srcinfo, matchrepos) # now check if all matched repos built successfully srcrepos = self.get_buildsuccess_repos(src_project, dst_project, src_srcinfo.package, src_srcinfo.verifymd5) if srcrepos is None: raise NotReadyYet(src_project, src_srcinfo.package, "no build success") if not srcrepos: raise NoBuildSuccess(src_project, src_srcinfo.package, src_srcinfo.verifymd5) for mr in matchrepos: if not (mr.srcrepo, arch) in srcrepos: self.logger.error("%s/%s had no build success" % (mr.srcrepo, arch)) raise NoBuildSuccess(src_project, src_srcinfo.package, src_srcinfo.verifymd5) return matchrepos # common with repochecker def _md5_disturl(self, disturl): """Get the md5 from the DISTURL from a RPM file.""" return os.path.basename(disturl).split('-')[0] def disturl_matches_md5(self, disturl, md5): if self._md5_disturl(disturl) != md5: return False return True # this is a bit magic. OBS allows to take the disturl md5 from the package # and query the source info for that. We will then get the verify md5 that # belongs to that md5. def disturl_matches(self, disturl, prj, srcinfo): md5 = self._md5_disturl(disturl) info = self.get_sourceinfo(prj, srcinfo.package, rev=md5) self.logger.debug(pformat(srcinfo)) self.logger.debug(pformat(info)) if info.verifymd5 == srcinfo.verifymd5: return True return False def compute_fetchlist(self, prj, pkg, srcinfo, repo, arch): """ scan binary rpms of the specified repo for libraries. Returns a set of packages to fetch and the libraries found """ self.logger.debug('scanning %s/%s %s/%s' % (prj, pkg, repo, arch)) headers = self._fetchcpioheaders(prj, pkg, repo, arch) missing_debuginfo = set() lib_packages = dict() # pkgname -> set(lib file names) pkgs = dict() # pkgname -> cpiohdr, rpmhdr lib_aliases = dict() for rpmfn, h in headers: # skip src rpm if h['sourcepackage']: continue pkgname = h['name'] if pkgname.endswith('-32bit') or pkgname.endswith('-64bit'): # -32bit and -64bit packages are just repackaged, so # we skip them and only check the original one. continue self.logger.debug(pkgname) if not self.disturl_matches(h['disturl'], prj, srcinfo): raise DistUrlMismatch(h['disturl'], srcinfo) pkgs[pkgname] = (rpmfn, h) if debugpkg_re.match(pkgname): continue for fn, mode, lnk in zip(h['filenames'], h['filemodes'], h['filelinktos']): if so_re.match(fn): if S_ISREG(mode): self.logger.debug('found lib: %s' % fn) lib_packages.setdefault(pkgname, set()).add(fn) elif S_ISLNK(mode) and lnk is not None: alias = os.path.basename(fn) libname = os.path.basename(lnk) self.logger.debug('found alias: %s -> %s' % (alias, libname)) lib_aliases.setdefault(libname, set()).add(alias) fetchlist = set() liblist = dict() # check whether debug info exists for each lib for pkgname in sorted(lib_packages.keys()): dpkgname = pkgname + '-debuginfo' if not dpkgname in pkgs: missing_debuginfo.add((prj, pkg, repo, arch, pkgname)) continue # check file list of debuginfo package rpmfn, h = pkgs[dpkgname] files = set(h['filenames']) ok = True for lib in lib_packages[pkgname]: fn = '/usr/lib/debug%s.debug' % lib if not fn in files: missing_debuginfo.add((prj, pkg, repo, arch, pkgname, lib)) ok = False if ok: fetchlist.add(pkgs[pkgname][0]) fetchlist.add(rpmfn) liblist.setdefault(lib, set()) libname = os.path.basename(lib) if libname in lib_aliases: liblist[lib] |= lib_aliases[libname] if missing_debuginfo: self.logger.error('missing debuginfo: %s' % pformat(missing_debuginfo)) raise MissingDebugInfo(missing_debuginfo) return fetchlist, liblist