def process_package_add_or_edit(copr, source_type_text, package=None, data=None): if not flask.g.user.can_edit(copr): raise InsufficientRightsException( "You are not allowed to add or edit packages in this copr.") formdata = data or flask.request.form try: if package and data: formdata = data.copy() for key in package.source_json_dict.keys() - data.keys(): value = package.source_json_dict[key] add_function = formdata.setlist if type( value) == list else formdata.add add_function(key, value) form = forms.get_package_form_cls_by_source_type_text( source_type_text)(formdata, meta={ 'csrf': False }) except UnknownSourceTypeException: raise LegacyApiError( "Unsupported package source type {source_type_text}".format( source_type_text=source_type_text)) if form.validate_on_submit(): if not package: try: package = PackagesLogic.add(flask.app.g.user, copr.main_dir, form.package_name.data) except InsufficientRightsException: raise LegacyApiError("Insufficient permissions.") except DuplicateException: raise LegacyApiError( "Package {0} already exists in copr {1}.".format( form.package_name.data, copr.full_name)) try: source_type = helpers.BuildSourceEnum(source_type_text) except KeyError: source_type = helpers.BuildSourceEnum("scm") package.source_type = source_type package.source_json = form.source_json if "webhook_rebuild" in formdata: package.webhook_rebuild = form.webhook_rebuild.data if "max_builds" in formdata: package.max_builds = form.max_builds.data db.session.add(package) db.session.commit() else: raise LegacyApiError(form.errors) return flask.jsonify({ "output": "ok", "message": "Create or edit operation was successful.", "package": package.to_dict(), })
def process_package_edit(copr, package_name, view): if request.form["source_type"] == "git_and_tito": form = forms.PackageFormTitoFactory form_var = "form_tito" elif request.form["source_type"] == "mock_scm": form = forms.PackageFormMockFactory form_var = "form_mock" else: raise Exception("Wrong source type") form = form.create_form_cls()() if form.validate_on_submit(): package = PackagesLogic.get(copr.id, package_name).first() package.source_type = helpers.BuildSourceEnum(form.source_type.data) if package.source_type == helpers.BuildSourceEnum("git_and_tito"): package.source_json = json.dumps({ "git_url": form.git_url.data, "git_branch": form.git_branch.data, "git_dir": form.git_directory.data, "tito_test": form.tito_test.data }) elif package.source_type == helpers.BuildSourceEnum("mock_scm"): package.source_json = json.dumps({ "scm_type": form.scm_type.data, "scm_url": form.scm_url.data, "scm_branch": form.scm_branch.data, "spec": form.spec.data }) db.session.add(package) db.session.commit() return render_package_edit(copr, package_name, view, **{form_var: form})
def batch_build(cls, user, copr, packages, chroot_names=None, **build_options): new_builds = [] batch = models.Batch() db.session.add(batch) for package in packages: git_hashes = {} skip_import = False source_build = None if (package.source_type == helpers.BuildSourceEnum('upload') or package.source_type == helpers.BuildSourceEnum('link')): source_build = package.last_build() if not source_build or not source_build.build_chroots[ 0].git_hash: raise exceptions.NoPackageSourceException( "Could not get latest git hash for {}".format( package.name)) for chroot_name in chroot_names: git_hashes[chroot_name] = source_build.build_chroots[ 0].git_hash skip_import = True new_build = builds_logic.BuildsLogic.create_new( user, copr, package.source_type, package.source_json, chroot_names, git_hashes=git_hashes, skip_import=skip_import, batch=batch, **build_options) if source_build: new_build.package_id = source_build.package_id new_build.pkg_version = source_build.pkg_version new_builds.append(new_build) return new_builds
def create_new_from_custom(cls, user, copr, script, script_chroot=None, script_builddeps=None, script_resultdir=None, chroot_names=None, copr_dirname=None, **kwargs): """ :type user: models.User :type copr: models.Copr :type script: str :type script_chroot: str :type script_builddeps: str :type script_resultdir: str :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("custom") source_dict = { 'script': script, 'chroot': script_chroot, 'builddeps': script_builddeps, 'resultdir': script_resultdir, } return cls.create_new(user, copr, source_type, json.dumps(source_dict), chroot_names, copr_dirname=copr_dirname, **kwargs)
def create_new_from_url(cls, user, copr, url, chroot_names=None, copr_dirname=None, **build_options): """ :type user: models.User :type copr: models.Copr :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("link") source_json = json.dumps({"url": url}) srpm_url = None if url.endswith('.spec') else url return cls.create_new(user, copr, source_type, source_json, chroot_names, pkgs=url, srpm_url=srpm_url, copr_dirname=copr_dirname, **build_options)
def create_new_from_other_build(cls, user, copr, source_build, chroot_names=None, **build_options): skip_import = False git_hashes = {} if source_build.source_type == helpers.BuildSourceEnum('upload'): if source_build.repeatable: skip_import = True for chroot in source_build.build_chroots: git_hashes[chroot.name] = chroot.git_hash else: raise UnrepeatableBuildException( "Build sources were not fully imported into CoprDistGit.") build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options) build.package_id = source_build.package_id build.pkg_version = source_build.pkg_version build.resubmitted_from_id = source_build.id return build
def create_new_from_tito(cls, user, copr, git_url, git_dir, git_branch, tito_test, chroot_names=None, **build_options): """ :type user: models.User :type copr: models.Copr :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("git_and_tito") source_json = json.dumps({ "git_url": git_url, "git_dir": git_dir, "git_branch": git_branch, "tito_test": tito_test }) return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
def add(cls, user, copr_dir, package_name, source_type=helpers.BuildSourceEnum("unset"), source_json=json.dumps({})): users_logic.UsersLogic.raise_if_cant_build_in_copr( user, copr_dir.copr, "You don't have permissions to build in this copr.") if cls.exists(copr_dir.id, package_name).all(): raise exceptions.DuplicateException( "Project dir {} already has a package '{}'".format( copr_dir.full_name, package_name)) package = models.Package( name=package_name, copr=copr_dir.copr, copr_dir=copr_dir, source_type=source_type, source_json=source_json, ) db.session.add(package) return package
def create_new_from_mock(cls, user, copr, scm_type, scm_url, scm_branch, spec, chroot_names=None, **build_options): """ :type user: models.User :type copr: models.Copr :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("mock_scm") source_json = json.dumps({ "scm_type": scm_type, "scm_url": scm_url, "scm_branch": scm_branch, "spec": spec }) return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, chroot_names=None, copr_dirname=None, **build_options): """ :type user: models.User :type copr: models.Copr :param f_uploader(file_path): function which stores data at the given `file_path` :return: """ tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"]) tmp_name = os.path.basename(tmp) filename = secure_filename(orig_filename) file_path = os.path.join(tmp, filename) f_uploader(file_path) # make the pkg public pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format( baseurl=app.config["PUBLIC_COPR_BASE_URL"], tmp_dir=tmp_name, filename=filename) # create json describing the build source source_type = helpers.BuildSourceEnum("upload") source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name}) srpm_url = None if pkg_url.endswith('.spec') else pkg_url try: build = cls.create_new(user, copr, source_type, source_json, chroot_names, pkgs=pkg_url, srpm_url=srpm_url, copr_dirname=copr_dirname, **build_options) except Exception: shutil.rmtree(tmp) # todo: maybe we should delete in some cleanup procedure? raise return build
def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions, chroot_names=None, **build_options): """ :type user: models.User :type copr: models.Copr :type package_name: str :type version: str :type python_versions: List[str] :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("pypi") source_json = json.dumps({ "pypi_package_name": pypi_package_name, "pypi_package_version": pypi_package_version, "python_versions": python_versions }) return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
def process_package_add_or_edit(copr, source_type, package=None): try: form = forms.get_package_form_cls_by_source_type(source_type)( csrf_enabled=False) except UnknownSourceTypeException: raise LegacyApiError( "Unsupported package source type {source_type}".format( source_type=source_type)) if form.validate_on_submit(): if not package: try: package = PackagesLogic.add(flask.app.g.user, copr, form.package_name.data) except InsufficientRightsException: raise LegacyApiError("Insufficient permissions.") except DuplicateException: raise LegacyApiError( "Package {0} already exists in copr {1}.".format( form.package_name.data, copr.full_name)) package.source_type = helpers.BuildSourceEnum(form.source_type.data) package.webhook_rebuild = form.webhook_rebuild.data package.source_json = form.source_json db.session.add(package) db.session.commit() else: raise LegacyApiError(form.errors) return flask.jsonify({ "output": "ok", "message": "Create or edit operation was successful.", "package": package.to_dict(), })
def create_new_from_other_build(cls, user, copr, source_build, chroot_names=None, **build_options): skip_import = False git_hashes = {} if source_build.source_type == helpers.BuildSourceEnum('srpm_upload'): # I don't have the source # so I don't want to import anything, just rebuild what's in dist git skip_import = True for chroot in source_build.build_chroots: if not chroot.git_hash: # I got an old build from time we didn't use dist git # So I'll submit it as a new build using it's link skip_import = False git_hashes = None flask.flash("This build is not in Dist Git. Trying to import the package again.") break git_hashes[chroot.name] = chroot.git_hash build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, **build_options) build.package_id = source_build.package_id build.pkg_version = source_build.pkg_version return build
def process_save_package(copr, source_type_text, package_name, view, view_method, url_on_success): form = forms.get_package_form_cls_by_source_type_text(source_type_text)() if "reset" in flask.request.form: try: package = PackagesLogic.get(copr.main_dir.id, package_name)[0] except IndexError: flask.flash( "Package {package_name} does not exist in copr_dir {copr_dir}." .format(package_name, copr.main_dir.full_name)) return flask.redirect(url_on_success) # should be url_on_fail try: PackagesLogic.reset_package(flask.g.user, package) db.session.commit() except InsufficientRightsException as e: flask.flash(str(e)) return flask.redirect(url_on_success) # should be url_on_fail flask.flash("Package default source successfully reset.") return flask.redirect(url_on_success) if form.validate_on_submit(): try: if package_name: package = PackagesLogic.get(copr.main_dir.id, package_name)[0] else: package = PackagesLogic.add(flask.app.g.user, copr.main_dir, form.package_name.data) package.source_type = helpers.BuildSourceEnum(source_type_text) package.webhook_rebuild = form.webhook_rebuild.data package.source_json = form.source_json package.chroot_blacklist_raw = form.chroot_blacklist.data package.max_builds = form.max_builds.data db.session.add(package) db.session.commit() except (InsufficientRightsException, IndexError) as e: db.session.rollback() flask.flash(str(e), "error") else: flask.flash( "Package successfully saved" if package_name else "New package has been created.", "success") return flask.redirect(url_on_success) kwargs = { "coprname": copr.name, "package_name": package_name, "source_type_text": source_type_text, "form": form, } kwargs.update({"group_name": copr.group.name} if copr. is_a_group_project else {"username": copr.user.name}) return view_method(**kwargs)
def f_hook_package(self, f_users, f_coprs, f_mock_chroots, f_builds): self.c1.webhook_secret = str(uuid.uuid4()) self.db.session.add(self.c1) self.pHook = models.Package( copr=self.c1, copr_dir=self.c1_dir, name="hook-package", source_type=helpers.BuildSourceEnum('scm'))
def reset_package(cls, user, package): if not user.can_edit(package.copr): raise exceptions.InsufficientRightsException( "You are not allowed to reset package `{}`.".format( package.id)) package.source_json = json.dumps({}) package.source_type = helpers.BuildSourceEnum("unset") db.session.add(package)
def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, chroot_names=None, **build_options): """ :type user: models.User :type copr: models.Copr :param f_uploader(file_path): function which stores data at the given `file_path` :return: """ tmp = tempfile.mkdtemp(dir=app.config["SRPM_STORAGE_DIR"]) tmp_name = os.path.basename(tmp) filename = secure_filename(orig_filename) file_path = os.path.join(tmp, filename) f_uploader(file_path) # make the pkg public pkg_url = "https://{hostname}/tmp/{tmp_dir}/{srpm}".format( hostname=app.config["PUBLIC_COPR_HOSTNAME"], tmp_dir=tmp_name, srpm=filename) # check which chroots we need chroots = [] for chroot in copr.active_chroots: if chroot.name in chroot_names: chroots.append(chroot) # create json describing the build source source_type = helpers.BuildSourceEnum("srpm_upload") source_json = json.dumps({"tmp": tmp_name, "pkg": filename}) try: build = cls.add(user=user, pkgs=pkg_url, copr=copr, chroots=chroots, source_type=source_type, source_json=source_json, enable_net=build_options.get( "enabled_net", copr.build_enable_net)) if user.proven: if "timeout" in build_options: build.timeout = build_options["timeout"] except Exception: shutil.rmtree( tmp) # todo: maybe we should delete in some cleanup procedure? raise return build
def create_new_from_rubygems(cls, user, copr, gem_name, chroot_names=None, copr_dirname=None, **build_options): """ :type user: models.User :type copr: models.Copr :type gem_name: str :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("rubygems") source_json = json.dumps({"gem_name": gem_name}) return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
def to_dict(self, options=None, with_chroot_states=False): result = super(Build, self).to_dict(options) result["src_pkg"] = result["pkgs"] del result["pkgs"] del result["copr_id"] result['source_type'] = helpers.BuildSourceEnum(result['source_type']) result["state"] = self.state if with_chroot_states: result["chroots"] = {b.name: b.state for b in self.build_chroots} return result
def delete_local_srpm(cls, build): """ Deletes the source rpm locally stored for upload (if exists) """ # is it hosted on the copr frontend? if build.source_type == helpers.BuildSourceEnum("srpm_upload"): data = json.loads(build.source_json) tmp = data["tmp"] storage_path = app.config["SRPM_STORAGE_DIR"] try: shutil.rmtree(os.path.join(storage_path, tmp)) except: pass
def create_new_from_url(cls, user, copr, srpm_url, chroot_names=None, **build_options): """ :type user: models.User :type copr: models.Copr :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("srpm_link") source_json = json.dumps({"url": srpm_url}) return cls.create_new(user, copr, source_type, source_json, chroot_names, pkgs=srpm_url, **build_options)
def create_new_from_mock(cls, user, copr, scm_type, scm_url, scm_branch, spec, chroot_names=None, **build_options): """ :type user: models.User :type copr: models.Copr :type chroot_names: List[str] :rtype: models.Build """ if chroot_names is None: chroots = [c for c in copr.active_chroots] else: chroots = [] for chroot in copr.active_chroots: if chroot.name in chroot_names: chroots.append(chroot) source_type = helpers.BuildSourceEnum("mock_scm") source_json = json.dumps({ "scm_type": scm_type, "scm_url": scm_url, "scm_branch": scm_branch, "spec": spec }) # try: build = cls.add(user=user, pkgs="", copr=copr, chroots=chroots, source_type=source_type, source_json=source_json, enable_net=build_options.get("enabled_net", copr.build_enable_net)) if user.proven: if "timeout" in build_options: build.timeout = build_options["timeout"] return build
class Package(db.Model, helpers.Serializer): """ Represents a single package in a project. """ id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(100), nullable=False) # Source of the build: type identifier source_type = db.Column(db.Integer, default=helpers.BuildSourceEnum("unset")) # Source of the build: description in json, example: git link, srpm url, etc. source_json = db.Column(db.Text) # True if the package is built automatically via webhooks webhook_rebuild = db.Column(db.Boolean, default=False) # enable networking during a build process enable_net = db.Column(db.Boolean, default=False, server_default="0", nullable=False) # relations copr_id = db.Column(db.Integer, db.ForeignKey("copr.id")) copr = db.relationship("Copr", backref=db.backref("packages")) @property def dist_git_repo(self): if self.copr.is_a_group_project: return "@{}/{}/{}".format(self.copr.group.name, self.copr.name, self.name) else: return "{}/{}/{}".format(self.copr.owner.name, self.copr.name, self.name) @property def source_json_dict(self): return json.loads(self.source_json) @property def source_type_text(self): return helpers.BuildSourceEnum(self.source_type) @property def dist_git_url(self): if app.config["DIST_GIT_URL"]: return "{}/{}.git".format(app.config["DIST_GIT_URL"], self.dist_git_repo) return None
def get_copr_package_info_rows(): source_type = helpers.BuildSourceEnum(args.backend.lower()) if db.engine.url.drivername == "sqlite": placeholder = '?' true = '1' else: placeholder = '%s' true = 'true' rows = db.engine.execute( """ SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id FROM package LEFT OUTER JOIN build ON build.package_id = package.id WHERE package.source_type = {placeholder} AND package.webhook_rebuild = {true} AND (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id)); """.format(placeholder=placeholder, true=true), source_type) return rows
def get_build_record(task, short=False): if not task: return None build_record = None try: build_record = { "task_id": task.task_id, "build_id": task.build.id, "project_owner": task.build.copr.owner_name, "project_name": task.build.copr_name, "project_dirname": task.build.copr_dirname, "submitter": task.build.submitter[0], "sandbox": task.build.sandbox, "chroot": task.mock_chroot.name, "repos": task.build.repos, "memory_reqs": task.build.memory_reqs, "timeout": task.build.timeout, "enable_net": task.build.enable_net, "git_repo": task.build.package.dist_git_repo, "git_hash": task.git_hash, "source_type": helpers.BuildSourceEnum("scm"), "source_json": json.dumps( {'clone_url': task.build.package.dist_git_clone_url, 'committish': task.git_hash}), "fetch_sources_only": True, "package_name": task.build.package.name, "package_version": task.build.pkg_version, "uses_devel_repo": task.build.copr.devel_mode, } if short: return build_record build_config = BuildConfigLogic.generate_build_config(task.build.copr, task.mock_chroot.name) build_record["repos"] = build_config.get("repos") build_record["buildroot_pkgs"] = build_config.get("additional_packages") build_record["use_bootstrap_container"] = build_config.get("use_bootstrap_container") build_record["with_opts"] = build_config.get("with_opts") build_record["without_opts"] = build_config.get("without_opts") except Exception as err: app.logger.exception(err) return None return build_record
def process_copr_repeat_build(build_id, copr): build = ComplexLogic.get_build_safe(build_id) if not flask.g.user.can_build_in(build.copr): flask.flash("You are not allowed to repeat this build.") if build.source_type == helpers.BuildSourceEnum('srpm_upload'): # If the original build's source is 'srpm_upload', we will show only the # original build's chroots and skip import. available_chroots = build.chroots else: # For all other sources, we will show all chroots enabled in the project # and proceed with import. available_chroots = copr.active_chroots form = forms.BuildFormRebuildFactory.create_form_cls(available_chroots)( build_id=build_id, enable_net=build.enable_net) # remove all checkboxes by default for ch in available_chroots: field = getattr(form, ch.name) field.data = False chroot_to_build = request.args.get("chroot") app.logger.debug("got param chroot: {}".format(chroot_to_build)) if chroot_to_build: # set single checkbox if chroot query arg was provided if hasattr(form, chroot_to_build): getattr(form, chroot_to_build).data = True else: build_chroot_names = set(ch.name for ch in build.chroots) build_failed_chroot_names = set(ch.name for ch in build.get_chroots_by_status([ helpers.StatusEnum('failed'), helpers.StatusEnum('canceled'), ])) for ch in available_chroots: # check checkbox on all the chroots that have not been (successfully) built before if (ch.name not in build_chroot_names) or ( ch.name in build_failed_chroot_names): getattr(form, ch.name).data = True return flask.render_template("coprs/detail/add_build/rebuild.html", copr=copr, build=build, form=form)
def create_new_from_distgit(cls, user, copr, clone_url, branch, chroot_names=None, **build_options): """ :type user: models.User :type copr: models.Copr :type clone_url: str :type branch: str :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("distgit") source_json = json.dumps({"clone_url": clone_url, "branch": branch}) return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
def create_new_from_scm(cls, user, copr, scm_type, clone_url, committish='', subdirectory='', spec='', srpm_build_method='rpkg', chroot_names=None, copr_dirname=None, **build_options): """ :type user: models.User :type copr: models.Copr :type chroot_names: List[str] :rtype: models.Build """ source_type = helpers.BuildSourceEnum("scm") source_json = json.dumps({"type": scm_type, "clone_url": clone_url, "committish": committish, "subdirectory": subdirectory, "spec": spec, "srpm_build_method": srpm_build_method}) return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
def to_dict(self, with_latest_build=False, with_latest_succeeded_build=False, with_all_builds=False): package_dict = super(Package, self).to_dict() package_dict['source_type'] = helpers.BuildSourceEnum( package_dict['source_type']) if with_latest_build: build = self.last_build(successful=False) package_dict['latest_build'] = build.to_dict( with_chroot_states=True) if build else None if with_latest_succeeded_build: build = self.last_build(successful=True) package_dict['latest_succeeded_build'] = build.to_dict( with_chroot_states=True) if build else None if with_all_builds: package_dict['builds'] = [ build.to_dict(with_chroot_states=True) for build in reversed(self.builds) ] return package_dict
def get_for_webhook_rebuild(cls, copr_id, webhook_secret, clone_url, commits, ref_type, ref): clone_url_stripped = re.sub(r'(\.git)?/*$', '', clone_url) packages = (models.Package.query.join(models.Copr).filter( models.Copr.webhook_secret == webhook_secret).filter( models.Package.source_type == helpers.BuildSourceEnum("scm") ).filter(models.Package.copr_id == copr_id).filter( models.Package.webhook_rebuild == true()).filter( models.Package.source_json.contains(clone_url_stripped))) result = [] for package in packages: package_clone_url = package.source_json_dict.get('clone_url', '') package_clone_url_stripped = re.sub(r'(\.git)?/*$', '', package_clone_url) if package_clone_url_stripped != clone_url_stripped: continue if cls.commits_belong_to_package(package, commits, ref_type, ref): result += [package] return result