def test_mark_as_failed(self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): BuildsLogic.mark_as_failed(self.b1.id) BuildsLogic.mark_as_failed(self.b3.id) assert self.b1.status == helpers.StatusEnum("succeeded") assert self.b3.status == helpers.StatusEnum("failed") assert type(BuildsLogic.mark_as_failed(self.b3.id)) == models.Build
def dist_git_upload_completed(): """ Mark BuildChroot in a Build as uploaded, which means: - set it to pending state - set BuildChroot.git_hash - if it's the last BuildChroot in a Build: - delete local srpm BuildChroot is identified with task_id which is build id + git branch name - For example: 56-f22 -> build 55, chroots fedora-22-* """ result = {"updated": False} if "task_id" in flask.request.json: app.logger.debug(flask.request.data) task_id = flask.request.json["task_id"] build_chroots = BuildsLogic.get_chroots_from_dist_git_task_id(task_id) build = build_chroots[0].build # Is it OK? if "git_hash" in flask.request.json and "repo_name" in flask.request.json: git_hash = flask.request.json["git_hash"] repo_name = flask.request.json["repo_name"] pkg_name = flask.request.json["pkg_name"] pkg_version = flask.request.json["pkg_version"] # Now I need to assign a package to this build package = PackagesLogic.get(build.copr.id, pkg_name).first() if not package: package = PackagesLogic.add(build.copr.owner, build.copr, pkg_name) db.session.add(package) db.session.flush() build.package_id = package.id build.pkg_version = pkg_version for ch in build_chroots: ch.status = helpers.StatusEnum("pending") ch.git_hash = git_hash # Failed? elif "error" in flask.request.json: error_type = flask.request.json["error"] try: build.fail_type = helpers.FailTypeEnum(error_type) except KeyError: build.fail_type = helpers.FailTypeEnum("unknown_error") for ch in build_chroots: ch.status = helpers.StatusEnum("failed") # is it the last chroot? if not build.has_importing_chroot: BuildsLogic.delete_local_srpm(build) db.session.commit() result.update({"updated": True}) return flask.jsonify(result)
def test_delete_build_basic( self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): self.b1.pkgs = "http://example.com/copr-keygen-1.58-1.fc20.src.rpm" expected_dir = self.b1.result_dir_name self.db.session.add(self.b1) self.db.session.commit() expected_chroots_to_delete = set() for bchroot in self.b1_bc: expected_chroots_to_delete.add(bchroot.name) assert len(ActionsLogic.get_many().all()) == 0 BuildsLogic.delete_build(self.u1, self.b1) self.db.session.commit() assert len(ActionsLogic.get_many().all()) == 1 action = ActionsLogic.get_many().one() delete_data = json.loads(action.data) assert "chroots" in delete_data assert delete_data["result_dir_name"] == expected_dir assert expected_chroots_to_delete == set(delete_data["chroots"]) with pytest.raises(NoResultFound): BuildsLogic.get(self.b1.id).one()
def process_new_build_upload(copr, add_view, url_on_success): form = forms.BuildFormUploadFactory.create_form_cls(copr.active_chroots)() if form.validate_on_submit(): build_options = { "enable_net": form.enable_net.data, "timeout": form.timeout.data, } try: BuildsLogic.create_new_from_upload( flask.g.user, copr, f_uploader=lambda path: form.pkgs.data.save(path), orig_filename=form.pkgs.data.filename, chroot_names=form.selected_chroots, **build_options ) db.session.commit() except (ActionInProgressException, InsufficientRightsException) as e: db.session.rollback() flask.flash(str(e), "error") else: flask.flash("New build has been created.") return flask.redirect(url_on_success) else: return render_add_build_upload(copr, form, add_view)
def factory(**build_options): source_build = ComplexLogic.get_build_safe(build_id) BuildsLogic.create_new_from_other_build( flask.g.user, copr, source_build, chroot_names=form.selected_chroots, **build_options )
def process_new_build_pypi(copr, add_view, url_on_success): form = forms.BuildFormPyPIFactory(copr.active_chroots)() if form.validate_on_submit(): build_options = { "enable_net": form.enable_net.data, "timeout": form.timeout.data, } try: BuildsLogic.create_new_from_pypi( flask.g.user, copr, form.pypi_package_name.data, form.pypi_package_version.data, form.python_versions.data, form.selected_chroots, **build_options ) db.session.commit() except (ActionInProgressException, InsufficientRightsException) as e: db.session.rollback() flask.flash(str(e), "error") else: flask.flash("New build has been created.") return flask.redirect(url_on_success) else: return render_add_build_pypi(copr, form, add_view)
def webhooks_hello(copr_id, uuid): # For the documentation of the data we receive see: # https://developer.github.com/v3/activity/events/types/#pushevent try: copr = ComplexLogic.get_copr_by_id_safe(copr_id) except ObjectNotFound: return page_not_found("Project does not exist") if copr.webhook_secret != uuid: return access_restricted("This webhook is not valid") try: request_json = flask.request.json clone_url = request_json["repository"]["clone_url"] except KeyError: return "Bad Request", 400 if "commits" in request_json: commits = request_json["commits"] else: commits = [] packages = PackagesLogic.get_for_webhook_rebuild(copr_id, uuid, clone_url, commits) for package in packages: BuildsLogic.rebuild_package(package) db.session.commit() return "OK", 200
def reschedule_build_chroot(): response = {} if "build_id" in flask.request.json and "chroot" in flask.request.json: build = ComplexLogic.get_build_safe(flask.request.json["build_id"]) else: response["result"] = "bad request" response["msg"] = "Request missing `build_id` and/or `chroot`" return flask.jsonify(response) if build: if build.canceled: response["result"] = "noop" response["msg"] = "build was cancelled, ignoring" else: chroot = flask.request.json["chroot"] build_chroot = build.chroots_dict_by_name.get(chroot) run_statuses = set([StatusEnum("starting"), StatusEnum("running")]) if build_chroot and build_chroot.status in run_statuses: log.info("rescheduling build {} chroot: {}".format(build.id, build_chroot.name)) BuildsLogic.update_state_from_dict(build, { "chroot": chroot, "status": StatusEnum("pending") }) db.session.commit() response["result"] = "done" else: response["result"] = "noop" response["msg"] = "build is not in running states, ignoring" else: response["result"] = "noop" response["msg"] = "Build {} wasn't found".format(flask.request.json["build_id"]) return flask.jsonify(response)
def test_delete_build_as_admin( self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): self.b4.pkgs = "http://example.com/copr-keygen-1.58-1.fc20.src.rpm" expected_dir = self.b4.result_dir_name for bc in self.b4_bc: bc.status = StatusEnum("succeeded") bc.ended_on = time.time() self.u1.admin = True self.db.session.add_all(self.b4_bc) self.db.session.add(self.b4) self.db.session.add(self.u1) self.db.session.commit() expected_chroots_to_delete = set() for bchroot in self.b4_bc: expected_chroots_to_delete.add(bchroot.name) assert len(ActionsLogic.get_many().all()) == 0 BuildsLogic.delete_build(self.u1, self.b4) self.db.session.commit() assert len(ActionsLogic.get_many().all()) == 1 action = ActionsLogic.get_many().one() delete_data = json.loads(action.data) assert "chroots" in delete_data assert delete_data["result_dir_name"] == expected_dir assert expected_chroots_to_delete == set(delete_data["chroots"]) with pytest.raises(NoResultFound): BuildsLogic.get(self.b4.id).one()
def process_rebuild(copr, build_id, view, url_on_success): source_build = ComplexLogic.get_build_safe(build_id) form = forms.BuildFormRebuildFactory.create_form_cls(copr.active_chroots)() if form.validate_on_submit(): try: build_options = { "enable_net": form.enable_net.data, "timeout": form.timeout.data, } BuildsLogic.create_new_from_other_build( flask.g.user, copr, source_build, chroot_names=form.selected_chroots, **build_options ) except (ActionInProgressException, InsufficientRightsException) as e: flask.flash(str(e), "error") db.session.rollback() else: flask.flash("New build has been created", "success") db.session.commit() return flask.redirect(url_on_success) else: return render_add_build(copr, form, view)
def factory(**build_options): BuildsLogic.create_new_from_upload( flask.g.user, copr, f_uploader=lambda path: form.pkgs.data.save(path), orig_filename=form.pkgs.data.filename, chroot_names=form.selected_chroots, **build_options )
def factory(**build_options): BuildsLogic.create_new_from_rubygems( flask.g.user, copr, form.gem_name.data, form.selected_chroots, **build_options )
def factory(**build_options): pkgs = form.pkgs.data.split("\n") for pkg in pkgs: BuildsLogic.create_new_from_url( flask.g.user, copr, pkg, chroot_names=form.selected_chroots, **build_options ) for pkg in pkgs: flask.flash("New build has been created: {}".format(pkg), "success")
def factory(**build_options): BuildsLogic.create_new_from_pypi( flask.g.user, copr, form.pypi_package_name.data, form.pypi_package_version.data, form.python_versions.data, form.selected_chroots, **build_options )
def test_delete_build_exceptions( self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): self.db.session.commit() with pytest.raises(InsufficientRightsException): BuildsLogic.delete_build(self.u1, self.b4) self.b1_bc[0].status = "running" self.db.session.add(self.b1_bc[0]) self.db.session.commit() with pytest.raises(ActionInProgressException): BuildsLogic.delete_build(self.u1, self.b1)
def factory(**build_options): BuildsLogic.create_new_from_tito( flask.g.user, copr, form.git_url.data, form.git_directory.data, form.git_branch.data, form.tito_test.data, form.selected_chroots, **build_options )
def factory(**build_options): BuildsLogic.create_new_from_mock( flask.g.user, copr, form.scm_type.data, form.scm_url.data, form.scm_branch.data, form.spec.data, form.selected_chroots, **build_options )
def test_delete_build_no_chroots_to_clean( self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): for bchroot in self.b1_bc: bchroot.status = helpers.StatusEnum("skipped") self.db.session.commit() assert len(ActionsLogic.get_many().all()) == 0 BuildsLogic.delete_build(self.u1, self.b1) self.db.session.commit() assert len(ActionsLogic.get_many().all()) == 0
def test_add_error_on_multiply_src(self, f_users, f_coprs, f_mock_chroots, f_db): params = dict( user=self.u1, pkgs="blah blah", copr=self.c1, repos="repos", timeout=5000) with pytest.raises(MalformedArgumentException): BuildsLogic.add(**params)
def dist_git_importing_queue(): """ Return list of builds that are waiting for dist git to import the sources. """ builds_list = [] for task in BuildsLogic.get_build_importing_queue().limit(200): copr = task.build.copr # we are using fake username's here if copr.is_a_group_project: user_name = u"@{}".format(copr.group.name) else: user_name = copr.user.name task_dict = { "task_id": task.import_task_id, "user": user_name, "project": task.build.copr.name, "branch": helpers.chroot_to_branch(task.mock_chroot.name), "source_type": task.build.source_type, "source_json": task.build.source_json, } if task_dict not in builds_list: builds_list.append(task_dict) response_dict = {"builds": builds_list} return flask.jsonify(response_dict)
def reschedule_all_running(): """ Add-hoc handle. Remove after implementation of persistent task handling in copr-backend """ to_reschedule = \ BuildsLogic.get_build_tasks(StatusEnum("starting")).all() + \ BuildsLogic.get_build_tasks(StatusEnum("running")).all() if to_reschedule: for build_chroot in to_reschedule: build_chroot.status = StatusEnum("pending") db.session.add(build_chroot) db.session.commit() return "OK", 200
def copr_new_build_pypi(copr): form = forms.BuildFormPyPIFactory(copr.active_chroots)(csrf_enabled=False) # TODO: automatically prepopulate all form fields with their defaults if not form.python_versions.data: form.python_versions.data = form.python_versions.default # are there any arguments in POST which our form doesn't know? if any([post_key not in form.__dict__.keys() for post_key in flask.request.form.keys()]): raise LegacyApiError("Unknown arguments passed (non-existing chroot probably)") if not form.validate_on_submit(): raise LegacyApiError("Invalid request: bad request parameters: {0}".format(form.errors)) # create a new build try: build = BuildsLogic.create_new_from_pypi( flask.g.user, copr, form.pypi_package_name.data, form.pypi_package_version.data, form.python_versions.data, form.selected_chroots, ) db.session.commit() except (ActionInProgressException, InsufficientRightsException) as e: raise LegacyApiError("Invalid request: {}".format(e)) output = {"output": "ok", "ids": [build.id], "message": "Build was added to {0}.".format(copr.name)} return flask.jsonify(output)
def copr_new_build_upload(copr): form = forms.BuildFormUploadFactory(copr.active_chroots)(csrf_enabled=False) # are there any arguments in POST which our form doesn't know? if any([post_key not in form.__dict__.keys() for post_key in flask.request.form.keys()]): raise LegacyApiError("Unknown arguments passed (non-existing chroot probably)") if not form.validate_on_submit(): raise LegacyApiError("Invalid request: bad request parameters") filename = secure_filename(form.pkgs.data.filename) # create a new build try: build = BuildsLogic.create_new_from_upload( flask.g.user, copr, f_uploader=lambda path: form.pkgs.data.save(path), orig_filename=filename, chroot_names=form.selected_chroots, ) db.session.commit() except (ActionInProgressException, InsufficientRightsException) as e: raise LegacyApiError("Invalid request: {}".format(e)) output = {"output": "ok", "ids": [build.id], "message": "Build was added to {0}.".format(copr.name)} return flask.jsonify(output)
def create_new_build(): return BuildsLogic.create_new_from_upload( flask.g.user, copr, f_uploader=lambda path: form.pkgs.data.save(path), orig_filename=secure_filename(form.pkgs.data.filename), chroot_names=form.selected_chroots, )
def main(): updated_packages = get_updated_packages(get_updates_messages()) loginfo('Updated packages according to datagrepper: {0}'.format(updated_packages)) for row in get_copr_package_info_rows(): source_json = json.loads(row.source_json) source_package_name = source_json['pypi_package_name'].lower() source_python_versions = source_json['python_versions'] latest_build_version = row.pkg_version loginfo('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(source_package_name, row.package_id, row.copr_id)) if source_package_name in updated_packages: new_updated_version = updated_packages[source_package_name] logdebug('source_package_name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(source_package_name, latest_build_version, new_updated_version)) if not latest_build_version or not re.match(new_updated_version, latest_build_version): # if the last build's package version is "different" from new remote package version, rebuild try: copr = CoprsLogic.get_by_id(row.copr_id)[0] except Exception as e: logexception(e) continue if args.backend.lower() == 'pypi': loginfo('Launching pypi build for package of source name: {0}, package_id: {1}, copr_id: {2}, user_id: {3}'.format(source_package_name, row.package_id, copr.id, copr.user.id)) build = BuildsLogic.create_new_from_pypi(copr.user, copr, source_package_name, new_updated_version, source_python_versions, chroot_names=None) else: raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend)) db.session.commit() loginfo('Launched build id {0}'.format(build.id))
def create_new_build(): return BuildsLogic.create_new_from_rubygems( flask.g.user, copr, form.gem_name.data, form.selected_chroots, )
def test_build_queue_3(self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): for build_chroots in [self.b1_bc, self.b2_bc, self.b3_bc, self.b4_bc]: for build_chroot in build_chroots: build_chroot.status = 0 self.db.session.commit() data = BuildsLogic.get_build_task_queue().all() assert len(data) == 0
def test_add_only_adds_active_chroots(self, f_users, f_coprs, f_builds, f_mock_chroots, f_db): self.mc2.is_active = False self.db.session.commit() b = BuildsLogic.add(self.u2, "blah", self.c2) self.db.session.commit() assert b.chroots[0].name == self.mc3.name
def create_new_build(): # create separate build for each package pkgs = form.pkgs.data.split("\n") return [BuildsLogic.create_new_from_url( flask.g.user, copr, srpm_url=pkg, chroot_names=form.selected_chroots, ) for pkg in pkgs]
def waiting(): """ Return list of waiting actions and builds. """ # models.Actions actions_list = [ action.to_dict(options={ "__columns_except__": ["result", "message", "ended_on"] }) for action in actions_logic.ActionsLogic.get_waiting() ] # tasks represented by models.BuildChroot with some other stuff builds_list = [] for task in BuildsLogic.get_build_task_queue().limit(200): try: copr = task.build.copr # we are using fake username's here if copr.is_a_group_project: user_name = u"@{}".format(copr.group.name) else: user_name = copr.user.name record = { "task_id": task.task_id, "build_id": task.build.id, "project_owner": user_name, "project_name": task.build.copr.name, "submitter": task.build.user.name if task.build.user else None, # there is no user for webhook builds "pkgs": task.build.pkgs, # TODO to be removed "chroot": task.mock_chroot.name, "repos": task.build.repos, "memory_reqs": task.build.memory_reqs, "timeout": task.build.timeout, "enable_net": task.build.enable_net, "git_repo": task.build.package.dist_git_repo, "git_hash": task.git_hash, "git_branch": helpers.chroot_to_branch(task.mock_chroot.name), "package_name": task.build.package.name, "package_version": task.build.pkg_version } copr_chroot = CoprChrootsLogic.get_by_name_safe(task.build.copr, task.mock_chroot.name) if copr_chroot: record["buildroot_pkgs"] = copr_chroot.buildroot_pkgs else: record["buildroot_pkgs"] = "" builds_list.append(record) except Exception as err: app.logger.exception(err) response_dict = {"actions": actions_list, "builds": builds_list} return flask.jsonify(response_dict)
def run(self): with db_session_scope(): BuildsLogic.clean_old_builds()
def test_add_raises_if_copr_has_unfinished_actions(self, f_users, f_coprs, f_actions, f_db): with pytest.raises(ActionInProgressException): b = BuildsLogic.add(self.u1, "blah", self.c1) self.db.session.rollback()
def test_build_queue_6(self, f_users, f_coprs, f_mock_chroots, f_db): self.db.session.commit() data = BuildsLogic.get_pending_build_tasks().all() assert len(data) == 0
def build(self, copr, new_update_version): return BuildsLogic.create_new_from_rubygems(copr.user, copr, self.name, chroot_names=None)
def factory(**build_options): BuildsLogic.create_new_from_mock(flask.g.user, copr, form.scm_type.data, form.scm_url.data, form.scm_branch.data, form.spec.data, form.selected_chroots, **build_options)
def factory(**build_options): BuildsLogic.create_new_from_pypi( flask.g.user, copr, form.pypi_package_name.data, form.pypi_package_version.data, form.spec_template.data, form.python_versions.data, form.selected_chroots, **build_options)
def copr_build_icon(build_id): return send_build_icon(BuildsLogic.get_by_id(int(build_id)).first())
def run(self): BuildsLogic.delete_orphaned_builds() PackagesLogic.delete_orphaned_packages()
def dist_git_upload_completed(): """ Mark BuildChroot in a Build as uploaded, which means: - set it to pending state - set BuildChroot.git_hash - if it's the last BuildChroot in a Build: - delete local srpm BuildChroot is identified with task_id which is build id + git branch name - For example: 56-f22 -> build 55, chroots fedora-22-* """ result = {"updated": False} if "task_id" in flask.request.json: app.logger.debug(flask.request.data) task_id = flask.request.json["task_id"] build_chroots = BuildsLogic.get_chroots_from_dist_git_task_id(task_id) build = build_chroots[0].build # Is it OK? if "git_hash" in flask.request.json and "repo_name" in flask.request.json: git_hash = flask.request.json["git_hash"] pkg_name = flask.request.json["pkg_name"] pkg_version = flask.request.json["pkg_version"] # Now I need to assign a package to this build if not PackagesLogic.get(build.copr.id, pkg_name).first(): try: package = PackagesLogic.add(build.copr.user, build.copr, pkg_name, build.source_type, build.source_json) db.session.add(package) db.session.commit() except (sqlalchemy.exc.IntegrityError, exceptions.DuplicateException) as e: db.session.rollback() package = PackagesLogic.get(build.copr.id, pkg_name).first() build.package_id = package.id build.pkg_version = pkg_version for ch in build_chroots: if ch.status == helpers.StatusEnum("importing"): ch.status = helpers.StatusEnum("pending") ch.git_hash = git_hash # Failed? elif "error" in flask.request.json: error_type = flask.request.json["error"] try: build.fail_type = helpers.FailTypeEnum(error_type) except KeyError: build.fail_type = helpers.FailTypeEnum("unknown_error") for ch in build_chroots: ch.status = helpers.StatusEnum("failed") # is it the last chroot? if not build.has_importing_chroot: BuildsLogic.delete_local_srpm(build) db.session.commit() result.update({"updated": True}) return flask.jsonify(result)
def render_user_info(user): graph = BuildsLogic.get_small_graph_data('30min') return flask.render_template("user_info.html", user=user, tasks_info=ComplexLogic.get_queue_sizes(), graph=graph)
def build(self, copr, new_updated_version): return BuildsLogic.create_new_from_pypi(copr.user, copr, self.name, new_updated_version, self.python_versions, chroot_names=None)
def test_build_queue_2(self, f_users, f_coprs, f_mock_chroots, f_db): self.db.session.commit() data = BuildsLogic.get_build_importing_queue().all() assert len(data) == 0
def split_one(orig_build_id): from coprs import db from coprs import models from coprs.logic.builds_logic import BuildsLogic from coprs.models import BuildChroot def clone_build(build): other = models.Build( pkgs=build.pkgs, built_packages=build.built_packages, pkg_version=build.pkg_version, canceled=build.canceled, repos=build.repos, submitted_on=build.submitted_on, started_on=build.started_on, ended_on=build.ended_on, results=build.results, memory_reqs=build.memory_reqs, timeout=build.timeout, enable_net=build.enable_net, user=build.user, copr=build.copr, ) return other def set_src_pkg(build_, src_pkg_): db.session.rollback( ) # some dirty state in SQLalchemy, no idea how to do correctly build_.pkgs = src_pkg_ db.session.add(build_) db.session.commit() log.info("Fixed spaces for build id={}, src_pkg={}".format( build_.id, src_pkg_)) build = BuildsLogic.get(orig_build_id).one() log.info("Start splitting build: {}, pkgs: {}".format( build.id, build.pkgs)) src_pkg_list = [] for mb_pkg in build.pkgs.strip().split(" "): src_pkg_list.append(mb_pkg) if len(src_pkg_list) == 0: log.error("> Got build with empty pkgs: id={}".format(build.id)) return if len(src_pkg_list) == 1: log.info("> Got build with one pkg in pkgs, id={}, pkgs={}".format( build.id, build.pkgs)) set_src_pkg(build, src_pkg_list[0]) return new_builds = [] new_build_chroots = [] for src_pkg in src_pkg_list: log.info("> Processing {} package".format(src_pkg)) new_build = clone_build(build) new_build.pkgs = src_pkg for bc in build.build_chroots: log.info("> > Copying chroot {}".format(bc.name)) new_bc = BuildChroot(build=new_build, mock_chroot=bc.mock_chroot, status=bc.status) new_build_chroots.append(new_bc) new_builds.append(new_build) log.info("> Finished build split for id: {}. Doing commit".format( build.id)) db.session.rollback( ) # some dirty state in SQLalchemy, no idea how to do correctly db.session.add_all(new_build_chroots) db.session.add_all(new_builds) for bc in build.build_chroots: db.session.delete(bc) db.session.delete(build) db.session.commit() log.info("> New build objects were created ") log.info("> Build {} deleted ".format(build.id))
def factory(**build_options): BuildsLogic.create_new_from_rubygems(flask.g.user, copr, form.gem_name.data, form.selected_chroots, **build_options)