def dist_git_upload_completed():
"""
Mark BuildChroot in a Build as uploaded, which means:
- set it to pending state
- set BuildChroot.git_hash
- if it's the last BuildChroot in a Build:
- delete local srpm
BuildChroot is identified with task_id which is build id + git branch name
- For example: 56-f22 -> build 55, chroots fedora-22-*
"""
result = {"updated": False}
if "task_id" in flask.request.json:
app.logger.debug(flask.request.data)
task_id = flask.request.json["task_id"]
build_chroots = BuildsLogic.get_chroots_from_dist_git_task_id(task_id)
build = build_chroots[0].build
# Is it OK?
if "git_hash" in flask.request.json and "repo_name" in flask.request.json:
git_hash = flask.request.json["git_hash"]
pkg_name = flask.request.json["pkg_name"]
pkg_version = flask.request.json["pkg_version"]
# Now I need to assign a package to this build
package = PackagesLogic.get(build.copr.id, pkg_name).first()
if not package:
package = PackagesLogic.add(build.copr.owner, build.copr,
pkg_name)
db.session.add(package)
db.session.flush()
build.package_id = package.id
build.pkg_version = pkg_version
for ch in build_chroots:
ch.status = helpers.StatusEnum("pending")
ch.git_hash = git_hash
# Failed?
elif "error" in flask.request.json:
error_type = flask.request.json["error"]
try:
build.fail_type = helpers.FailTypeEnum(error_type)
except KeyError:
build.fail_type = helpers.FailTypeEnum("unknown_error")
for ch in build_chroots:
ch.status = helpers.StatusEnum("failed")
# is it the last chroot?
if not build.has_importing_chroot:
BuildsLogic.delete_local_srpm(build)
db.session.commit()
result.update({"updated": True})
return flask.jsonify(result)
def test_mark_as_failed(self, f_users, f_coprs, f_mock_chroots, f_builds, f_db):
BuildsLogic.mark_as_failed(self.b1.id)
BuildsLogic.mark_as_failed(self.b3.id)
assert self.b1.status == helpers.StatusEnum("succeeded")
assert self.b3.status == helpers.StatusEnum("failed")
assert type(BuildsLogic.mark_as_failed(self.b3.id)) == models.Build
def get_build_task_queue(cls):
"""
Returns BuildChroots which are - waiting to be built or
- older than 2 hours and unfinished
"""
# todo: filter out build without package
query = (
models.BuildChroot.query.join(models.Build).
filter(models.Build.canceled == false()).filter(
or_(
models.BuildChroot.status == helpers.StatusEnum("pending"),
models.BuildChroot.status == helpers.StatusEnum(
"starting"),
and_(
# We are moving ended_on to the BuildChroot, now it should be reliable,
# so we don't want to reschedule failed chroots
# models.BuildChroot.status.in_([
# # Bug 1206562 - Cannot delete Copr because it incorrectly thinks
# # there are unfinished builds. Solution: `failed` but unfinished
# # (ended_on is null) builds should be rescheduled.
# # todo: we need to be sure that correct `failed` set is set together wtih `ended_on`
# helpers.StatusEnum("running"),
# helpers.StatusEnum("failed")
#]),
models.BuildChroot.status == helpers.StatusEnum(
"running"),
models.BuildChroot.started_on <
int(time.time() - 1.1 * MAX_BUILD_TIMEOUT),
models.BuildChroot.ended_on.is_(None)))))
query = query.order_by(models.BuildChroot.build_id.asc())
return query
def process_copr_repeat_build(build_id, copr):
build = ComplexLogic.get_build_safe(build_id)
if not flask.g.user.can_build_in(build.copr):
flask.flash("You are not allowed to repeat this build.")
form = forms.BuildFormRebuildFactory.create_form_cls(build.chroots)(
build_id=build_id,
enable_net=build.enable_net,
)
# remove all checkboxes by default
for ch in build.chroots:
field = getattr(form, ch.name)
field.data = False
chroot_to_build = request.args.get("chroot")
app.logger.debug("got param chroot: {}".format(chroot_to_build))
if chroot_to_build:
# set single checkbox if chroot query arg was provided
if hasattr(form, chroot_to_build):
getattr(form, chroot_to_build).data = True
else:
# set checkbox on the failed chroots
chroots_to_select = set(ch.name for ch in build.get_chroots_by_status([
helpers.StatusEnum('failed'),
helpers.StatusEnum('canceled'),
]))
for ch in build.chroots:
if ch.name in chroots_to_select:
getattr(form, ch.name).data = True
return flask.render_template("coprs/detail/add_build/rebuild.html",
copr=copr,
build=build,
form=form)
def mark_as_failed(cls, build_id):
"""
Marks build as failed on all its non-finished chroots
"""
build = cls.get(build_id).one()
chroots = filter(lambda x: x.status != helpers.StatusEnum("succeeded"), build.build_chroots)
for chroot in chroots:
chroot.status = helpers.StatusEnum("failed")
return build
def importing():
tasks = builds_logic.BuildsLogic.get_build_tasks(
helpers.StatusEnum("importing"), background=False).limit(200)
bg_tasks_cnt = builds_logic.BuildsLogic.get_build_tasks(
helpers.StatusEnum("importing"), background=True).count()
return flask.render_template("status/importing.html",
number=len(list(tasks)),
bg_tasks_cnt=bg_tasks_cnt,
tasks=tasks)
def repeatable(self):
"""
Find out if this build is repeatable.
Build is repeatable only if it's not pending, starting or running
"""
return self.status not in [
helpers.StatusEnum("pending"),
helpers.StatusEnum("starting"),
helpers.StatusEnum("running"),
]
def status(self):
"""
Return build status according to build status of its chroots
"""
if self.canceled:
return helpers.StatusEnum("canceled")
for state in [
"failed", "running", "starting", "pending", "succeeded",
"skipped"
]:
if helpers.StatusEnum(state) in self.chroot_states:
return helpers.StatusEnum(state)
def rebuild_package(cls, package):
build = models.Build(user=None,
pkgs=None,
package_id=package.id,
copr=package.copr,
repos=package.copr.repos,
source_type=package.source_type,
source_json=package.source_json,
submitted_on=int(time.time()),
enable_net=package.enable_net,
timeout=DEFAULT_BUILD_TIMEOUT)
db.session.add(build)
chroots = package.copr.active_chroots
status = helpers.StatusEnum("importing")
for chroot in chroots:
buildchroot = models.BuildChroot(build=build,
status=status,
mock_chroot=chroot,
git_hash=None)
db.session.add(buildchroot)
return build
def last_modified(cls, copr):
""" Get build datetime (as epoch) of last successful build
:arg copr: object of copr
"""
builds = cls.get_multiple_by_copr(copr)
last_build = (builds.join(models.BuildChroot).filter(
(models.BuildChroot.status == helpers.StatusEnum("succeeded"))
| (models.BuildChroot.status == helpers.StatusEnum("skipped"))).
filter(models.Build.ended_on.isnot(None)).order_by(
models.Build.ended_on.desc())).first()
if last_build:
return last_build.ended_on
else:
return None
def get_build_task_queue(cls):
"""
Returns BuildChroots which are - waiting to be built or
- older than 2 hours and unfinished
"""
query = models.BuildChroot.query.join(models.Build).filter(
or_(
models.BuildChroot.status == helpers.StatusEnum("pending"),
models.BuildChroot.status == helpers.StatusEnum("starting"),
and_(
models.BuildChroot.status == helpers.StatusEnum("running"),
models.Build.started_on <
int(time.time() - 1.1 * MAX_BUILD_TIMEOUT),
models.Build.ended_on.is_(None))))
query = query.order_by(models.BuildChroot.build_id.asc())
return query
def cancelable(self):
"""
Find out if this build is cancelable.
Build is cancelabel only when it's pending (not started)
"""
return self.status == helpers.StatusEnum("pending")
def get_build_task(cls):
query = (models.BuildChroot.query.join(models.Build)
.filter(models.Build.canceled == false())
.filter(or_(
models.BuildChroot.status == helpers.StatusEnum("pending"),
and_(
models.BuildChroot.status == helpers.StatusEnum("running"),
models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT),
models.BuildChroot.ended_on.is_(None)
)
))
.filter(or_(
models.BuildChroot.last_deferred.is_(None),
models.BuildChroot.last_deferred < int(time.time() - DEFER_BUILD_SECONDS)
))
).order_by(models.Build.is_background.asc(), models.BuildChroot.build_id.asc())
return query.first()
def test_monitor_logic(self, f_users, f_coprs, f_mock_chroots_many,
f_build_many_chroots, f_db):
copr = self.c1
md = BuildsMonitorLogic.get_monitor_data(copr)
results = md["packages"][-1][-1]
mchroots = md["chroots"]
for chr, res in zip(mchroots, results):
assert helpers.StatusEnum(self.status_by_chroot[chr]) == res[1]
def state(self):
"""
Return text representation of status of this build chroot
"""
if self.status is not None:
return helpers.StatusEnum(self.status)
return "unknown"
def get_build_importing_queue(cls):
"""
Returns BuildChroots which are waiting to be uploaded to dist git
"""
query = (models.BuildChroot.query.join(
models.Build).filter(models.Build.canceled == false()).filter(
models.BuildChroot.status == helpers.StatusEnum("importing")))
query = query.order_by(models.BuildChroot.build_id.asc())
return query
def process_copr_repeat_build(build_id, copr):
build = ComplexLogic.get_build_safe(build_id)
if not flask.g.user.can_build_in(build.copr):
flask.flash("You are not allowed to repeat this build.")
if build.source_type == helpers.BuildSourceEnum('srpm_upload'):
# If the original build's source is 'srpm_upload', we will show only the
# original build's chroots and skip import.
available_chroots = build.chroots
else:
# For all other sources, we will show all chroots enabled in the project
# and proceed with import.
available_chroots = copr.active_chroots
form = forms.BuildFormRebuildFactory.create_form_cls(available_chroots)(
build_id=build_id, enable_net=build.enable_net)
# remove all checkboxes by default
for ch in available_chroots:
field = getattr(form, ch.name)
field.data = False
chroot_to_build = request.args.get("chroot")
app.logger.debug("got param chroot: {}".format(chroot_to_build))
if chroot_to_build:
# set single checkbox if chroot query arg was provided
if hasattr(form, chroot_to_build):
getattr(form, chroot_to_build).data = True
else:
build_chroot_names = set(ch.name for ch in build.chroots)
build_failed_chroot_names = set(ch.name
for ch in build.get_chroots_by_status([
helpers.StatusEnum('failed'),
helpers.StatusEnum('canceled'),
]))
for ch in available_chroots:
# check checkbox on all the chroots that have not been (successfully) built before
if (ch.name not in build_chroot_names) or (
ch.name in build_failed_chroot_names):
getattr(form, ch.name).data = True
return flask.render_template("coprs/detail/add_build/rebuild.html",
copr=copr,
build=build,
form=form)
def copr_repeat_build(username, coprname, build_id, page=1):
build = builds_logic.BuildsLogic.get(build_id).first()
copr = coprs_logic.CoprsLogic.get(
flask.g.user, username=username, coprname=coprname).first()
if not build:
return page_not_found(
"Build with id {0} does not exist.".format(build_id))
if not copr:
return page_not_found(
"Copr {0}/{1} does not exist.".format(username, coprname))
if not flask.g.user.can_build_in(build.copr):
flask.flash("You are not allowed to repeat this build.")
form = forms.BuildFormFactory.create_form_cls(copr.active_chroots)(
pkgs=build.pkgs, enable_net=build.enable_net,
)
# remove all checkboxes by default
for ch in copr.active_chroots:
field = getattr(form, ch.name)
field.data = False
chroot_to_build = request.args.get("chroot")
app.logger.debug("got param chroot: {}".format(chroot_to_build))
if chroot_to_build:
# set single checkbox if chroot query arg was provided
if hasattr(form, chroot_to_build):
getattr(form, chroot_to_build).data = True
else:
# set checkbox on the failed chroots
chroots_to_select = set(ch.name for ch in build.get_chroots_by_status([
helpers.StatusEnum('failed'), helpers.StatusEnum('canceled'),
]))
for ch in build.chroots:
if ch.name in chroots_to_select:
getattr(form, ch.name).data = True
return flask.render_template("coprs/detail/add_build.html",
copr=copr, form=form)
def test_delete_build_no_chroots_to_clean(self, f_users, f_coprs,
f_mock_chroots, f_builds, f_db):
for bchroot in self.b1_bc:
bchroot.status = helpers.StatusEnum("skipped")
self.db.session.commit()
assert len(ActionsLogic.get_many().all()) == 0
BuildsLogic.delete_build(self.u1, self.b1)
self.db.session.commit()
assert len(ActionsLogic.get_many().all()) == 0
def get_monitor_data(cls, copr):
copr_packages = sorted(copr.packages, key=lambda pkg: pkg.name)
packages = []
for pkg in copr_packages:
chroots = {}
for ch in copr.active_chroots:
# todo: move to ComplexLogic
query = (models.BuildChroot.query.join(models.Build).filter(
models.Build.package_id == pkg.id).filter(
models.BuildChroot.mock_chroot_id == ch.id).filter(
models.BuildChroot.status != helpers.StatusEnum(
"canceled")))
build = query.order_by(
models.BuildChroot.build_id.desc()).first()
chroots[ch.name] = build
packages.append({"package": pkg, "build_chroots": chroots})
return packages
def coprs_show(page=1):
query = coprs_logic.CoprsLogic.get_multiple(
flask.g.user, with_mock_chroots=False)
paginator = helpers.Paginator(query, query.count(), page)
coprs = paginator.sliced_query
# flask.g.user is none when no user is logged - showing builds from everyone
users_builds = builds_logic.BuildsLogic.get_recent_tasks(flask.g.user, 5)
waiting_tasks = len(list(builds_logic.BuildsLogic.get_build_task_queue()))
running_tasks = len(list(builds_logic.BuildsLogic
.get_build_tasks(helpers.StatusEnum("running"))))
return flask.render_template("coprs/show.html",
coprs=coprs,
paginator=paginator,
waiting_tasks=waiting_tasks,
running_tasks=running_tasks,
users_builds=users_builds)
def test_delete_build_some_chroots(self, f_users, f_coprs, f_mock_chroots,
f_builds, f_db):
expected_chroots_to_delete = set(
[self.b1_bc[0].name, self.b1_bc[-1].name])
for bchroot in self.b1_bc[1:-1]:
bchroot.status = helpers.StatusEnum("skipped")
self.db.session.commit()
assert len(ActionsLogic.get_many().all()) == 0
BuildsLogic.delete_build(self.u1, self.b1)
self.db.session.commit()
assert len(ActionsLogic.get_many().all()) == 1
action = ActionsLogic.get_many().one()
delete_data = json.loads(action.data)
assert "chroots" in delete_data
assert expected_chroots_to_delete == set(delete_data["chroots"])
with pytest.raises(NoResultFound):
BuildsLogic.get(self.b1.id).one()
class BuildChroot(db.Model, helpers.Serializer):
"""
Representation of Build<->MockChroot relation
"""
mock_chroot_id = db.Column(db.Integer,
db.ForeignKey("mock_chroot.id"),
primary_key=True)
mock_chroot = db.relationship("MockChroot", backref=db.backref("builds"))
build_id = db.Column(db.Integer,
db.ForeignKey("build.id"),
primary_key=True)
build = db.relationship("Build", backref=db.backref("build_chroots"))
status = db.Column(db.Integer, default=helpers.StatusEnum("pending"))
@property
def name(self):
"""
Textual representation of name of this chroot
"""
return self.mock_chroot.name
@property
def state(self):
"""
Return text representation of status of this build chroot
"""
if self.status is not None:
return helpers.StatusEnum(self.status)
return "unknown"
def __str__(self):
return "<BuildChroot: {}>".format(self.to_dict())
def add(cls,
user,
pkgs,
copr,
source_type=None,
source_json=None,
repos=None,
chroots=None,
timeout=None,
enable_net=True,
git_hashes=None,
skip_import=False):
if chroots is None:
chroots = []
coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
copr,
"Can't build while there is an operation in progress: {action}")
users_logic.UsersLogic.raise_if_cant_build_in_copr(
user, copr, "You don't have permissions to build in this copr.")
if not repos:
repos = copr.repos
# todo: eliminate pkgs and this check
if " " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs:
raise exceptions.MalformedArgumentException(
"Trying to create a build using src_pkg "
"with bad characters. Forgot to split?")
# just temporary to keep compatibility
if not source_type or not source_json:
source_type = helpers.BuildSourceEnum("srpm_link")
source_json = json.dumps({"url": pkgs})
build = models.Build(
user=user,
pkgs=pkgs,
copr=copr,
repos=repos,
source_type=source_type,
source_json=source_json,
submitted_on=int(time.time()),
enable_net=bool(enable_net),
)
if timeout:
build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
db.session.add(build)
# add BuildChroot object for each active (or selected) chroot
# this copr is assigned to
if not chroots:
chroots = copr.active_chroots
status = helpers.StatusEnum("importing")
if skip_import:
status = StatusEnum("pending")
for chroot in chroots:
git_hash = None
if git_hashes:
git_hash = git_hashes.get(chroot.name)
buildchroot = models.BuildChroot(build=build,
status=status,
mock_chroot=chroot,
git_hash=git_hash)
db.session.add(buildchroot)
return build
def running():
tasks = builds_logic.BuildsLogic.get_build_tasks(helpers.StatusEnum("running"))
return flask.render_template("status/running.html",
number=len(list(tasks)),
tasks=tasks)
def ensure_rebuild(dstdb):
for chroot in dstdb.session.query(models.BuildChroot).all():
chroot.status = helpers.StatusEnum(
"pending" if chroot.git_hash else "importing")
def has_unfinished_chroot(self):
return helpers.StatusEnum("pending") in self.chroot_states or \
helpers.StatusEnum("starting") in self.chroot_states or \
helpers.StatusEnum("running") in self.chroot_states
def has_pending_chroot(self):
# FIXME bad name
# used when checking if the repo is initialized and results can be set
# i think this is the only purpose - check
return helpers.StatusEnum("pending") in self.chroot_states or \
helpers.StatusEnum("starting") in self.chroot_states
def rebuild_failed(dstdb):
for i in range(1, 5):
builds = dstdb.session.query(models.BuildChroot).filter_by(
status=helpers.StatusEnum("failed")).all()
for chroot in builds:
chroot.status = helpers.StatusEnum("pending")
def importing():
tasks = builds_logic.BuildsLogic.get_build_tasks(
helpers.StatusEnum("importing")).limit(200)
return flask.render_template("status/importing.html",
number=len(list(tasks)),
tasks=tasks)
