def compare(): parser = argparse.ArgumentParser() parser.add_argument('--info-repo', help="use local rdoinfo repo instead of" "fetching default one using rdopkg") options, args = parser.parse_known_args(sys.argv[1:]) packages = getpackages(local_info_repo=options.info_repo, tags=cp.get("DEFAULT", "tags")) compare_details = {} # Each argument is a ":" seperate filename:title, this filename is the # sqlite db file and the title is whats used in the dable being displayed table_header = ["Name", "Out of Sync"] for dbdetail in args: dbfilename, dbtitle = dbdetail.split(":") table_header.extend((dbtitle + " upstream", dbtitle + " spec")) session = getSession('sqlite:///%s' % dbfilename) for package in packages: package_name = package["name"] compare_details.setdefault(package_name, [package_name, " "]) last_success = getCommits(session, project=package_name, with_status="SUCCESS").first() if last_success: compare_details[package_name].extend( (last_success.commit_hash[:8], last_success.distro_hash[:8])) else: compare_details[package_name].extend(("None", "None")) session.close() table = PrettyTable(table_header) for name, compare_detail in compare_details.items(): if len(set(compare_detail)) > 4: compare_detail[1] = "*" table.add_row(compare_detail) print(table)
def genreports(cp, packages, options): global session session = getSession('sqlite:///commits.sqlite') # Generate report of the last 300 package builds target = cp.get("DEFAULT", "target") src = cp.get("DEFAULT", "source") reponame = cp.get("DEFAULT", "reponame") templatedir = cp.get("DEFAULT", "templatedir") datadir = cp.get("DEFAULT", "datadir") css_file = os.path.join(templatedir, 'stylesheets/styles.css') # configure jinja and filters jinja_env = jinja2.Environment( loader=jinja2.FileSystemLoader([templatedir])) jinja_env.filters["strftime"] = _jinja2_filter_strftime jinja_env.filters["get_commit_url"] = \ partial(_jinja2_filter_get_commit_url, packages=packages) # generate build report commits = getCommits(session, without_status="RETRY", limit=300) jinja_template = jinja_env.get_template("report.j2") content = jinja_template.render(reponame=reponame, src=src, target=target, commits=commits) shutil.copy2(css_file, os.path.join(datadir, "repos", "styles.css")) report_file = os.path.join(cp.get("DEFAULT", "datadir"), "repos", "report.html") with open(report_file, "w") as fp: fp.write(content) # Generate status report if options.head_only: msg = "(all commit not built)" else: msg = "" pkgs = [] # Find the most recent successfull build # then report on failures since then for package in packages: name = package["name"] commits = getCommits(session, project=name, limit=1) # No builds if commits.count() == 0: continue pkgs.append(package) last_build = commits.first() package["last_build"] = last_build # last build was successul if last_build.status == "SUCCESS": continue # Retrieve last successful build commits = getCommits(session, project=name, with_status="SUCCESS", limit=1) # No successful builds if commits.count() == 0: commits = getCommits(session, project=name, with_status="FAILED", order="asc") package["first_failure"] = commits.first() package["days"] = -1 continue last_success = commits.first() last_success_dt = last_success.dt_build commits = getCommits(session, project=name, with_status="FAILED", order="asc", limit=None) commits = commits.filter(Commit.dt_build > last_success_dt) package["first_failure"] = commits.first() package["days"] = (datetime.now() - datetime.fromtimestamp(last_success_dt)).days pkgs = sorted(pkgs, key=itemgetter("name")) jinja_template = jinja_env.get_template("status_report.j2") content = jinja_template.render(msg=msg, reponame=reponame, src=src, target=target, pkgs=pkgs) report_file = os.path.join(cp.get("DEFAULT", "datadir"), "repos", "status_report.html") with open(report_file, "w") as fp: fp.write(content)
def test_first_failed(self): commits = db.getCommits(self.session, project="python-pysaml2", with_status="FAILED", order="asc") self.assertEqual(commits.count(), 1) self.assertEqual(commits.first().id, 5874)
def test_last_two(self): commits = db.getCommits(self.session, project="python-pysaml2", limit=2) self.assertEqual(commits.count(), 2) self.assertEqual([c.id for c in commits], [7835, 7834])
def test_last_without_retry(self): commits = db.getCommits(self.session, project="python-tripleoclient", without_status="RETRY") self.assertEqual(commits.count(), 1) self.assertEqual(commits.first().id, 7696)
def test_last_success(self): commits = db.getCommits(self.session, project="python-tripleoclient", with_status="SUCCESS") self.assertEqual(commits.count(), 1) self.assertEqual(commits.first().id, 7696)
def test_no_results(self): commits = db.getCommits(self.session, project="dummy") self.assertEqual(commits.count(), 0) self.assertEqual(commits.first(), None)
def test_defaults(self): commits = db.getCommits(self.session) self.assertEqual(commits.count(), 1) self.assertEqual(commits.first().id, 7873)
def test_first_failed_since(self): commits = db.getCommits(self.session, project="python-alembic", with_status="FAILED", order="asc", since="1442487440") self.assertEqual(commits.count(), 1) self.assertEqual(commits.first().id, 6230)
def build(cp, packages, commit, env_vars, dev_mode, use_public, bootstrap): # Set the build timestamp to now commit.dt_build = int(time()) scriptsdir = os.path.realpath(cp.get("DEFAULT", "scriptsdir")) run(os.path.join(scriptsdir, "build_rpm_wrapper.sh"), cp, commit, env_vars, dev_mode, use_public, bootstrap) datadir = os.path.realpath(cp.get("DEFAULT", "datadir")) yumrepodir = os.path.join("repos", commit.getshardedcommitdir()) yumrepodir_abs = os.path.join(datadir, yumrepodir) commit_hash = commit.commit_hash project_name = commit.project_name built_rpms = [] for rpm in os.listdir(yumrepodir_abs): if rpm.endswith(".rpm"): built_rpms.append(os.path.join(yumrepodir, rpm)) if not built_rpms: raise Exception("No rpms built for %s" % project_name) notes = "OK" if not os.path.isfile(os.path.join(yumrepodir_abs, "installed")): logger.error('Build failed. See logs at: %s/%s/' % (datadir, yumrepodir)) raise Exception("Error installing %s" % project_name) shafile = open(os.path.join(yumrepodir_abs, "versions.csv"), "w") shafile.write("Project,Source Repo,Source Sha,Dist Repo,Dist Sha," "Status,Last Success Timestamp\n") failures = 0 for otherproject in packages: otherprojectname = otherproject["name"] if otherprojectname == project_name: # Output sha's this project dumpshas2file(shafile, commit, otherproject["upstream"], otherproject["master-distgit"], "SUCCESS", commit.dt_build) continue # Output sha's of all other projects represented in this repo last_success = getCommits(session, project=otherprojectname, with_status="SUCCESS").first() last_processed = getLastProcessedCommit(session, otherprojectname, 'INVALID STATE') if last_success: for rpm in last_success.rpms.split(","): rpm_link_src = os.path.join(yumrepodir_abs, os.path.split(rpm)[1]) os.symlink(os.path.relpath(os.path.join(datadir, rpm), yumrepodir_abs), rpm_link_src) last = last_success else: last = last_processed if last: dumpshas2file(shafile, last, otherproject["upstream"], otherproject["master-distgit"], last_processed.status, last.dt_build) if last_processed.status != 'SUCCESS': failures += 1 else: failures += 1 shafile.close() sh.createrepo(yumrepodir_abs) with open(os.path.join( yumrepodir_abs, "%s.repo" % cp.get("DEFAULT", "reponame")), "w") as fp: fp.write("[%s]\nname=%s-%s-%s\nbaseurl=%s/%s\nenabled=1\n" "gpgcheck=0\npriority=1" % (cp.get("DEFAULT", "reponame"), cp.get("DEFAULT", "reponame"), project_name, commit_hash, cp.get("DEFAULT", "baseurl"), commit.getshardedcommitdir())) dirnames = ['current'] if failures == 0: dirnames.append('consistent') else: logger.info('%d packages not built correctly: not updating the ' 'consistent symlink' % failures) for dirname in dirnames: target_repo_dir = os.path.join(datadir, "repos", dirname) os.symlink(os.path.relpath(yumrepodir_abs, os.path.join(datadir, "repos")), target_repo_dir + "_") os.rename(target_repo_dir + "_", target_repo_dir) return built_rpms, notes
def genreports(cp, packages, options): global session session = getSession('sqlite:///commits.sqlite') # Generate report of the last 300 package builds target = cp.get("DEFAULT", "target") src = cp.get("DEFAULT", "source") reponame = cp.get("DEFAULT", "reponame") templatedir = cp.get("DEFAULT", "templatedir") datadir = cp.get("DEFAULT", "datadir") css_file = os.path.join(templatedir, 'stylesheets/styles.css') # configure jinja and filters jinja_env = jinja2.Environment( loader=jinja2.FileSystemLoader([templatedir])) jinja_env.filters["strftime"] = _jinja2_filter_strftime jinja_env.filters["get_commit_url"] = \ partial(_jinja2_filter_get_commit_url, packages=packages) # generate build report commits = getCommits(session, without_status="RETRY", limit=300) jinja_template = jinja_env.get_template("report.j2") content = jinja_template.render(reponame=reponame, src=src, target=target, commits=commits) shutil.copy2(css_file, os.path.join(datadir, "repos", "styles.css")) report_file = os.path.join(cp.get("DEFAULT", "datadir"), "repos", "report.html") with open(report_file, "w") as fp: fp.write(content) # Generate status report if options.head_only: msg = "(all commit not built)" else: msg = "" pkgs = [] # Find the most recent successfull build # then report on failures since then for package in packages: name = package["name"] commits = getCommits(session, project=name, limit=1) # No builds if commits.count() == 0: continue pkgs.append(package) last_build = commits.first() package["last_build"] = last_build # last build was successul if last_build.status == "SUCCESS": continue # Retrieve last successful build commits = getCommits(session, project=name, with_status="SUCCESS", limit=1) # No successful builds if commits.count() == 0: commits = getCommits(session, project=name, with_status="FAILED", order="asc") package["first_failure"] = commits.first() package["days"] = -1 continue last_success = commits.first() last_success_dt = last_success.dt_build commits = getCommits(session, project=name, with_status="FAILED", order="asc", limit=None) commits = commits.filter(Commit.dt_build > last_success_dt) package["first_failure"] = commits.first() package["days"] = (datetime.now() - datetime.fromtimestamp(last_success_dt)).days pkgs = sorted(pkgs, key=itemgetter("name")) jinja_template = jinja_env.get_template("status_report.j2") content = jinja_template.render(msg=msg, reponame=reponame, src=src, target=target, pkgs=pkgs) report_file = os.path.join(cp.get("DEFAULT", "datadir"), "repos", "status_report.html") with open(report_file, "w") as fp: fp.write(content)