def compare():
parser = argparse.ArgumentParser()
parser.add_argument('--info-repo',
help="use local rdoinfo repo instead of"
"fetching default one using rdopkg")
options, args = parser.parse_known_args(sys.argv[1:])
packages = getpackages(local_info_repo=options.info_repo,
tags=cp.get("DEFAULT", "tags"))
compare_details = {}
# Each argument is a ":" seperate filename:title, this filename is the
# sqlite db file and the title is whats used in the dable being displayed
table_header = ["Name", "Out of Sync"]
for dbdetail in args:
dbfilename, dbtitle = dbdetail.split(":")
table_header.extend((dbtitle + " upstream", dbtitle + " spec"))
session = getSession('sqlite:///%s' % dbfilename)
for package in packages:
package_name = package["name"]
compare_details.setdefault(package_name, [package_name, " "])
last_success = getCommits(session, project=package_name,
with_status="SUCCESS").first()
if last_success:
compare_details[package_name].extend(
(last_success.commit_hash[:8],
last_success.distro_hash[:8]))
else:
compare_details[package_name].extend(("None", "None"))
session.close()
table = PrettyTable(table_header)
for name, compare_detail in compare_details.items():
if len(set(compare_detail)) > 4:
compare_detail[1] = "*"
table.add_row(compare_detail)
print(table)
def genreports(cp, packages, options):
global session
session = getSession('sqlite:///commits.sqlite')
# Generate report of the last 300 package builds
target = cp.get("DEFAULT", "target")
src = cp.get("DEFAULT", "source")
reponame = cp.get("DEFAULT", "reponame")
templatedir = cp.get("DEFAULT", "templatedir")
datadir = cp.get("DEFAULT", "datadir")
css_file = os.path.join(templatedir, 'stylesheets/styles.css')
# configure jinja and filters
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader([templatedir]))
jinja_env.filters["strftime"] = _jinja2_filter_strftime
jinja_env.filters["get_commit_url"] = \
partial(_jinja2_filter_get_commit_url, packages=packages)
# generate build report
commits = getCommits(session, without_status="RETRY", limit=300)
jinja_template = jinja_env.get_template("report.j2")
content = jinja_template.render(reponame=reponame,
src=src,
target=target,
commits=commits)
shutil.copy2(css_file, os.path.join(datadir, "repos", "styles.css"))
report_file = os.path.join(cp.get("DEFAULT", "datadir"), "repos",
"report.html")
with open(report_file, "w") as fp:
fp.write(content)
# Generate status report
if options.head_only:
msg = "(all commit not built)"
else:
msg = ""
pkgs = []
# Find the most recent successfull build
# then report on failures since then
for package in packages:
name = package["name"]
commits = getCommits(session, project=name, limit=1)
# No builds
if commits.count() == 0:
continue
pkgs.append(package)
last_build = commits.first()
package["last_build"] = last_build
# last build was successul
if last_build.status == "SUCCESS":
continue
# Retrieve last successful build
commits = getCommits(session,
project=name,
with_status="SUCCESS",
limit=1)
# No successful builds
if commits.count() == 0:
commits = getCommits(session,
project=name,
with_status="FAILED",
order="asc")
package["first_failure"] = commits.first()
package["days"] = -1
continue
last_success = commits.first()
last_success_dt = last_success.dt_build
commits = getCommits(session,
project=name,
with_status="FAILED",
order="asc",
limit=None)
commits = commits.filter(Commit.dt_build > last_success_dt)
package["first_failure"] = commits.first()
package["days"] = (datetime.now() -
datetime.fromtimestamp(last_success_dt)).days
pkgs = sorted(pkgs, key=itemgetter("name"))
jinja_template = jinja_env.get_template("status_report.j2")
content = jinja_template.render(msg=msg,
reponame=reponame,
src=src,
target=target,
pkgs=pkgs)
report_file = os.path.join(cp.get("DEFAULT", "datadir"), "repos",
"status_report.html")
with open(report_file, "w") as fp:
fp.write(content)
def test_first_failed(self):
commits = db.getCommits(self.session, project="python-pysaml2",
with_status="FAILED", order="asc")
self.assertEqual(commits.count(), 1)
self.assertEqual(commits.first().id, 5874)
def test_last_two(self):
commits = db.getCommits(self.session, project="python-pysaml2",
limit=2)
self.assertEqual(commits.count(), 2)
self.assertEqual([c.id for c in commits], [7835, 7834])
def test_last_without_retry(self):
commits = db.getCommits(self.session, project="python-tripleoclient",
without_status="RETRY")
self.assertEqual(commits.count(), 1)
self.assertEqual(commits.first().id, 7696)
def test_last_success(self):
commits = db.getCommits(self.session, project="python-tripleoclient",
with_status="SUCCESS")
self.assertEqual(commits.count(), 1)
self.assertEqual(commits.first().id, 7696)
def test_no_results(self):
commits = db.getCommits(self.session, project="dummy")
self.assertEqual(commits.count(), 0)
self.assertEqual(commits.first(), None)
def test_defaults(self):
commits = db.getCommits(self.session)
self.assertEqual(commits.count(), 1)
self.assertEqual(commits.first().id, 7873)
def test_first_failed_since(self):
commits = db.getCommits(self.session, project="python-alembic",
with_status="FAILED", order="asc",
since="1442487440")
self.assertEqual(commits.count(), 1)
self.assertEqual(commits.first().id, 6230)
def build(cp, packages, commit, env_vars, dev_mode, use_public, bootstrap):
# Set the build timestamp to now
commit.dt_build = int(time())
scriptsdir = os.path.realpath(cp.get("DEFAULT", "scriptsdir"))
run(os.path.join(scriptsdir, "build_rpm_wrapper.sh"), cp, commit, env_vars,
dev_mode, use_public, bootstrap)
datadir = os.path.realpath(cp.get("DEFAULT", "datadir"))
yumrepodir = os.path.join("repos", commit.getshardedcommitdir())
yumrepodir_abs = os.path.join(datadir, yumrepodir)
commit_hash = commit.commit_hash
project_name = commit.project_name
built_rpms = []
for rpm in os.listdir(yumrepodir_abs):
if rpm.endswith(".rpm"):
built_rpms.append(os.path.join(yumrepodir, rpm))
if not built_rpms:
raise Exception("No rpms built for %s" % project_name)
notes = "OK"
if not os.path.isfile(os.path.join(yumrepodir_abs, "installed")):
logger.error('Build failed. See logs at: %s/%s/' % (datadir,
yumrepodir))
raise Exception("Error installing %s" % project_name)
shafile = open(os.path.join(yumrepodir_abs, "versions.csv"), "w")
shafile.write("Project,Source Repo,Source Sha,Dist Repo,Dist Sha,"
"Status,Last Success Timestamp\n")
failures = 0
for otherproject in packages:
otherprojectname = otherproject["name"]
if otherprojectname == project_name:
# Output sha's this project
dumpshas2file(shafile, commit, otherproject["upstream"],
otherproject["master-distgit"], "SUCCESS",
commit.dt_build)
continue
# Output sha's of all other projects represented in this repo
last_success = getCommits(session, project=otherprojectname,
with_status="SUCCESS").first()
last_processed = getLastProcessedCommit(session, otherprojectname,
'INVALID STATE')
if last_success:
for rpm in last_success.rpms.split(","):
rpm_link_src = os.path.join(yumrepodir_abs,
os.path.split(rpm)[1])
os.symlink(os.path.relpath(os.path.join(datadir, rpm),
yumrepodir_abs), rpm_link_src)
last = last_success
else:
last = last_processed
if last:
dumpshas2file(shafile, last, otherproject["upstream"],
otherproject["master-distgit"],
last_processed.status, last.dt_build)
if last_processed.status != 'SUCCESS':
failures += 1
else:
failures += 1
shafile.close()
sh.createrepo(yumrepodir_abs)
with open(os.path.join(
yumrepodir_abs, "%s.repo" % cp.get("DEFAULT", "reponame")),
"w") as fp:
fp.write("[%s]\nname=%s-%s-%s\nbaseurl=%s/%s\nenabled=1\n"
"gpgcheck=0\npriority=1" % (cp.get("DEFAULT", "reponame"),
cp.get("DEFAULT", "reponame"),
project_name, commit_hash,
cp.get("DEFAULT", "baseurl"),
commit.getshardedcommitdir()))
dirnames = ['current']
if failures == 0:
dirnames.append('consistent')
else:
logger.info('%d packages not built correctly: not updating the '
'consistent symlink' % failures)
for dirname in dirnames:
target_repo_dir = os.path.join(datadir, "repos", dirname)
os.symlink(os.path.relpath(yumrepodir_abs,
os.path.join(datadir, "repos")),
target_repo_dir + "_")
os.rename(target_repo_dir + "_", target_repo_dir)
return built_rpms, notes
def genreports(cp, packages, options):
global session
session = getSession('sqlite:///commits.sqlite')
# Generate report of the last 300 package builds
target = cp.get("DEFAULT", "target")
src = cp.get("DEFAULT", "source")
reponame = cp.get("DEFAULT", "reponame")
templatedir = cp.get("DEFAULT", "templatedir")
datadir = cp.get("DEFAULT", "datadir")
css_file = os.path.join(templatedir, 'stylesheets/styles.css')
# configure jinja and filters
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader([templatedir]))
jinja_env.filters["strftime"] = _jinja2_filter_strftime
jinja_env.filters["get_commit_url"] = \
partial(_jinja2_filter_get_commit_url, packages=packages)
# generate build report
commits = getCommits(session, without_status="RETRY", limit=300)
jinja_template = jinja_env.get_template("report.j2")
content = jinja_template.render(reponame=reponame,
src=src,
target=target,
commits=commits)
shutil.copy2(css_file, os.path.join(datadir, "repos", "styles.css"))
report_file = os.path.join(cp.get("DEFAULT", "datadir"),
"repos", "report.html")
with open(report_file, "w") as fp:
fp.write(content)
# Generate status report
if options.head_only:
msg = "(all commit not built)"
else:
msg = ""
pkgs = []
# Find the most recent successfull build
# then report on failures since then
for package in packages:
name = package["name"]
commits = getCommits(session, project=name, limit=1)
# No builds
if commits.count() == 0:
continue
pkgs.append(package)
last_build = commits.first()
package["last_build"] = last_build
# last build was successul
if last_build.status == "SUCCESS":
continue
# Retrieve last successful build
commits = getCommits(session, project=name, with_status="SUCCESS",
limit=1)
# No successful builds
if commits.count() == 0:
commits = getCommits(session, project=name, with_status="FAILED",
order="asc")
package["first_failure"] = commits.first()
package["days"] = -1
continue
last_success = commits.first()
last_success_dt = last_success.dt_build
commits = getCommits(session, project=name, with_status="FAILED",
order="asc", limit=None)
commits = commits.filter(Commit.dt_build > last_success_dt)
package["first_failure"] = commits.first()
package["days"] = (datetime.now() -
datetime.fromtimestamp(last_success_dt)).days
pkgs = sorted(pkgs, key=itemgetter("name"))
jinja_template = jinja_env.get_template("status_report.j2")
content = jinja_template.render(msg=msg,
reponame=reponame,
src=src,
target=target,
pkgs=pkgs)
report_file = os.path.join(cp.get("DEFAULT", "datadir"),
"repos", "status_report.html")
with open(report_file, "w") as fp:
fp.write(content)