def test_set_spec_ver_empty_changelog(tmp_path):
u_remote_path = tmp_path / "upstream_remote"
u_remote_path.mkdir(parents=True, exist_ok=True)
subprocess.check_call(["git", "init", "--bare", "."], cwd=u_remote_path)
u = tmp_path / "upstream_git"
shutil.copytree(EMPTY_CHANGELOG, u)
initiate_git_repo(u, tag="0.1.0")
with cwd(tmp_path):
c = get_test_config()
pc = get_local_package_config(str(u))
pc.upstream_project_url = str(u)
lp = LocalProject(working_dir=u)
ups = Upstream(c, pc, lp)
new_ver = "1.2.3"
ups.specfile.set_spec_version(version=new_ver, changelog_entry="- asdqwe")
assert ups.get_specfile_version() == new_ver
assert "%changelog" not in u.joinpath("beer.spec").read_text()
def test_working_dir():
flexmock(local_project).should_receive("is_git_repo").with_args(
"./local/directory/to/git").and_return(True)
flexmock(git).should_receive("Repo").with_args(
path="./local/directory/to/git").and_return(
flexmock(
active_branch=flexmock(name="branch"),
head=flexmock(is_detached=False),
remotes=[
flexmock(
name="origin",
url="https://server.git/my_namespace/package_name")
],
))
project = LocalProject(working_dir="./local/directory/to/git")
assert project
assert project.git_url == "https://server.git/my_namespace/package_name"
assert project.namespace == "my_namespace"
assert project.working_dir == "./local/directory/to/git"
assert project.git_repo
assert project.ref == "branch"
def test_pr_id_and_ref(tmp_path: Path):
""" p-s passes both ref and pr_id, we want to check out PR """
remote = tmp_path / "remote"
remote.mkdir()
create_new_repo(remote, ["--bare"])
upstream_git = tmp_path / "upstream_git"
upstream_git.mkdir()
initiate_git_repo(upstream_git, push=True, upstream_remote=str(remote))
# mimic github PR
pr_id = "123"
ref = (subprocess.check_output(["git", "rev-parse", "HEAD^"],
cwd=upstream_git).strip().decode())
local_tmp_branch = "asdqwe"
subprocess.check_call(["git", "branch", local_tmp_branch, ref],
cwd=upstream_git)
subprocess.check_call(
[
"git", "push", "origin",
f"{local_tmp_branch}:refs/pull/{pr_id}/head"
],
cwd=upstream_git,
)
subprocess.check_call(["git", "branch", "-D", local_tmp_branch],
cwd=upstream_git)
LocalProject(
working_dir=upstream_git,
offline=True,
pr_id=pr_id,
ref=ref,
git_service=GithubService(),
)
assert (subprocess.check_output(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
cwd=upstream_git).strip().decode() == f"pr/{pr_id}")
def convert(self, value, param, ctx):
try:
branch_name = None
if self.branch_param_name:
if self.branch_param_name in ctx.params:
branch_name = ctx.params[self.branch_param_name]
else: # use the default
for param in ctx.command.params:
if param.name == self.branch_param_name:
branch_name = param.default
remote_name = ctx.params.get(self.remote_param_name, None)
local_project = LocalProject(path_or_url=value,
ref=branch_name,
remote=remote_name)
if not local_project.working_dir and not local_project.git_url:
self.fail(
"Parameter is not an existing directory nor correct git url.",
param,
ctx,
)
return local_project
except Exception as ex:
self.fail(ex, param, ctx)
def run(self) -> TaskResults:
"""
Sync the upstream release to dist-git as a pull request.
"""
self.local_project = LocalProject(
git_project=self.project,
working_dir=self.service_config.command_handler_work_dir,
)
self.api = PackitAPI(self.service_config, self.job_config,
self.local_project)
errors = {}
default_dg_branch = self.api.dg.local_project.git_project.default_branch
for branch in get_branches(*self.job_config.metadata.dist_git_branches,
default=default_dg_branch):
try:
self.api.sync_release(dist_git_branch=branch,
tag=self.data.tag_name)
except Exception as ex:
# the archive has not been uploaded to PyPI yet
if FILE_DOWNLOAD_FAILURE in str(ex):
# retry for the archive to become available
logger.info(
f"We were not able to download the archive: {ex}")
# when the task hits max_retries, it raises MaxRetriesExceededError
# and the error handling code would be never executed
retries = self.task.request.retries
if retries < int(
getenv("CELERY_RETRY_LIMIT", DEFAULT_RETRY_LIMIT)):
logger.info(f"Retrying for the {retries + 1}. time...")
# throw=False so that exception is not raised and task
# is not retried also automatically
self.task.retry(exc=ex,
countdown=15 * 2**retries,
throw=False)
return TaskResults(
success=False,
details={
"msg":
"Task was retried, we were not able to download the archive."
},
)
sentry_integration.send_to_sentry(ex)
errors[branch] = str(ex)
if errors:
branch_errors = ""
for branch, err in sorted(
errors.items(), key=lambda branch_error: branch_error[0]):
err_without_new_lines = err.replace("\n", " ")
branch_errors += f"| `{branch}` | `{err_without_new_lines}` |\n"
msg_retrigger = MSG_RETRIGGER.format(job="update",
command="propose-downstream",
place="issue")
body_msg = (
f"Packit failed on creating pull-requests in dist-git:\n\n"
f"| dist-git branch | error |\n"
f"| --------------- | ----- |\n"
f"{branch_errors}\n\n"
f"{msg_retrigger}\n")
self.project.create_issue(
title=
f"[packit] Propose downstream failed for release {self.data.tag_name}",
body=body_msg,
)
return TaskResults(
success=False,
details={
"msg": "Propose downstream failed.",
"errors": errors
},
)
return TaskResults(success=True, details={})
def run(self) -> TaskResults:
"""
Sync the upstream release to dist-git as a pull request.
"""
self.local_project = LocalProject(
git_project=self.project,
working_dir=self.service_config.command_handler_work_dir,
)
self.api = PackitAPI(self.service_config, self.job_config,
self.local_project)
errors = {}
for branch in get_branches(*self.job_config.metadata.dist_git_branches,
default="master"):
try:
self.api.sync_release(dist_git_branch=branch,
version=self.data.tag_name)
except Exception as ex:
# the archive has not been uploaded to PyPI yet
if FILE_DOWNLOAD_FAILURE in str(ex):
# retry for the archive to become available
logger.info(
f"We were not able to download the archive: {ex}")
# when the task hits max_retries, it raises MaxRetriesExceededError
# and the error handling code would be never executed
retries = self.task.request.retries
if retries < RETRY_LIMIT:
logger.info(f"Retrying for the {retries + 1}. time...")
self.task.retry(exc=ex, countdown=15 * 2**retries)
sentry_integration.send_to_sentry(ex)
errors[branch] = str(ex)
if errors:
branch_errors = ""
for branch, err in sorted(
errors.items(), key=lambda branch_error: branch_error[0]):
err_without_new_lines = err.replace("\n", " ")
branch_errors += f"| `{branch}` | `{err_without_new_lines}` |\n"
body_msg = (
f"Packit failed on creating pull-requests in dist-git:\n\n"
f"| dist-git branch | error |\n"
f"| --------------- | ----- |\n"
f"{branch_errors}\n\n"
"You can re-trigger the update by adding `/packit propose-update`"
" to the issue comment.\n")
self.project.create_issue(
title=
f"[packit] Propose update failed for release {self.data.tag_name}",
body=body_msg,
)
return TaskResults(
success=False,
details={
"msg": "Propose update failed.",
"errors": errors
},
)
return TaskResults(success=True, details={})
def run(self) -> HandlerResults:
self.local_project = LocalProject(
git_project=self.project,
working_dir=self.config.command_handler_work_dir)
logger.info("Running testing farm")
r = BuildStatusReporter(self.project, self.event.commit_sha)
chroots = self.job.metadata.get("targets")
logger.debug(f"Testing farm chroots: {chroots}")
for chroot in chroots:
pipeline_id = str(uuid.uuid4())
logger.debug(f"Pipeline id: {pipeline_id}")
payload: dict = {
"pipeline": {
"id": pipeline_id
},
"api": {
"token": self.config.testing_farm_secret
},
}
logger.debug(f"Payload: {payload}")
stg = "-stg" if self.config.deployment == Deployment.stg else ""
copr_repo_name = (
f"packit/{self.project.namespace}-{self.project.repo}-"
f"{self.event.pr_id}{stg}")
payload["artifact"] = {
"repo-name": self.event.base_repo_name,
"repo-namespace": self.event.base_repo_namespace,
"copr-repo-name": copr_repo_name,
"copr-chroot": chroot,
"commit-sha": self.event.commit_sha,
"git-url": self.event.project_url,
"git-ref": self.base_ref,
}
logger.debug("Sending testing farm request...")
logger.debug(payload)
req = self.send_testing_farm_request(TESTING_FARM_TRIGGER_URL,
"POST", {},
json.dumps(payload))
logger.debug(f"Request sent: {req}")
if not req:
msg = "Failed to post request to testing farm API."
logger.debug("Failed to post request to testing farm API.")
r.report(
"failure",
msg,
None,
"",
check_names=PRCheckName.get_testing_farm_check(chroot),
)
return HandlerResults(success=False, details={"msg": msg})
else:
logger.debug(
f"Submitted to testing farm with return code: {req.status_code}"
)
"""
Response:
{
"id": "9fa3cbd1-83f2-4326-a118-aad59f5",
"success": true,
"url": "https://console-testing-farm.apps.ci.centos.org/pipeline/<id>"
}
"""
# success set check on pending
if req.status_code != 200:
# something went wrong
msg = req.json()["message"]
r.report(
"failure",
msg,
None,
check_names=PRCheckName.get_testing_farm_check(chroot),
)
return HandlerResults(success=False, details={"msg": msg})
r.report(
"pending",
"Tests are running ...",
None,
req.json()["url"],
check_names=PRCheckName.get_testing_farm_check(chroot),
)
return HandlerResults(success=True, details={})
def update_dist_git(
config: Config,
source_git: str,
dist_git: str,
upstream_ref: Optional[str],
pkg_tool: Optional[str],
message: Optional[str],
file: Optional[str],
):
"""Update a dist-git repository using content from a source-git repository
Update a dist-git repository with patches created from the commits
between <upstream_ref> and the current HEAD of the source-git repo.
This command, by default, performs only local operations and uses
the content of the source-git and dist-git repository as it is:
does not checkout branches or fetches remotes.
A commit in dist-git is created only if a commit message is provided with
--message or --file.
The source archives are retrieved from the upstream URLs specified in
the spec-file and uploaded to the lookaside cache in dist-git only if
'--pkg-tool' is specified.
Examples:
To update a dist-git repo from source-git without uploading the source-archive
to the lookaside cache and creating a commit with the updates, run:
\b
$ packit source-git update-dist-git src/curl rpms/curl
To also commit the changes and upload the source-archive to the lookaside-cache
specify -m and --pkg-tool:
\b
$ packit source-git update-dist-git -m'Update from source-git' \\
--pkg-tool fedpkg src/curl rpms/curl
"""
if message and file:
raise click.BadOptionUsage("-m",
"Option -m cannot be combined with -F.")
if pkg_tool and not which(pkg_tool):
raise click.BadOptionUsage(
"--pkg-tool", f"{pkg_tool} is not executable or in any path")
if file:
with click.open_file(file, "r") as fp:
message = fp.read()
source_git_path = pathlib.Path(source_git).resolve()
dist_git_path = pathlib.Path(dist_git).resolve()
package_config = get_local_package_config(
source_git_path, package_config_path=config.package_config_path)
api = PackitAPI(
config=config,
package_config=package_config,
upstream_local_project=LocalProject(working_dir=source_git_path,
offline=True),
downstream_local_project=LocalProject(working_dir=dist_git_path,
offline=True),
)
title, _, message = message.partition("\n\n") if message else (None, None,
None)
api.update_dist_git(
version=None,
upstream_ref=upstream_ref or package_config.upstream_ref,
# Add new sources if a pkg_tool was specified.
add_new_sources=bool(pkg_tool),
force_new_sources=False,
upstream_tag=None,
commit_title=title,
commit_msg=message,
sync_default_files=False,
pkg_tool=pkg_tool,
)
def api_instance_source_git(sourcegit_and_remote, distgit_and_remote):
sourcegit, _ = sourcegit_and_remote
distgit, _ = distgit_and_remote
up_lp = LocalProject(working_dir=sourcegit)
return mock_api_for_source_git(sourcegit, distgit, up_lp)
def __init__(
self,
local_project: LocalProject,
config: Config,
upstream_url: Optional[str] = None,
upstream_ref: Optional[str] = None,
dist_git_path: Optional[Path] = None,
dist_git_branch: Optional[str] = None,
fedora_package: Optional[str] = None,
centos_package: Optional[str] = None,
tmpdir: Optional[Path] = None,
):
"""
:param local_project: this source-git repo
:param config: global configuration
:param upstream_url: upstream repo URL we want to use as a base
:param upstream_ref: upstream git-ref to use as a base
:param dist_git_path: path to a local clone of a dist-git repo
:param dist_git_branch: branch in dist-git to use
:param fedora_package: pick up specfile and downstream sources from this fedora package
:param centos_package: pick up specfile and downstream sources from this centos package
:param tmpdir: path to a directory where temporary repos (upstream,
dist-git) will be cloned
"""
self.local_project = local_project
self.config = config
self.tmpdir = tmpdir or Path(tempfile.mkdtemp(prefix="packit-sg-"))
self._dist_git: Optional[DistGit] = None
self._primary_archive: Optional[Path] = None
self._upstream_ref: Optional[str] = upstream_ref
self.dist_git_branch = dist_git_branch
logger.info(
f"The source-git repo is going to be created in {local_project.working_dir}."
)
if dist_git_path:
(
self._dist_git,
self.centos_package,
self.fedora_package,
) = get_distgit_kls_from_repo(dist_git_path, config)
self.dist_git_path = dist_git_path
self.package_config = self.dist_git.package_config
else:
self.centos_package = centos_package
self.fedora_package = fedora_package
if centos_package:
self.package_config = PackageConfig(
downstream_package_name=centos_package)
elif fedora_package:
self.fedora_package = (self.fedora_package
or local_project.working_dir.name)
self.package_config = PackageConfig(
downstream_package_name=fedora_package)
else:
raise PackitException(
"Please tell us the name of the package in the downstream."
)
self.dist_git_path = self.tmpdir.joinpath(
self.package_config.downstream_package_name)
if upstream_url:
if Path(upstream_url).is_dir():
self.upstream_repo_path: Path = Path(upstream_url)
self.upstream_lp: LocalProject = LocalProject(
working_dir=self.upstream_repo_path)
else:
self.upstream_repo_path = self.tmpdir.joinpath(
f"{self.package_config.downstream_package_name}-upstream")
self.upstream_lp = LocalProject(
git_url=upstream_url, working_dir=self.upstream_repo_path)
else:
# $CWD is the upstream repo and we just need to pick
# downstream stuff
self.upstream_repo_path = self.local_project.working_dir
self.upstream_lp = self.local_project
def mock_remote_functionality(distgit: Path, upstream: Path):
def mocked_pr_create(*args, **kwargs):
return PullRequest(
title="",
id=42,
status=PRStatus.open,
url="",
description="",
author="",
source_branch="",
target_branch="",
created=datetime.datetime(1969, 11, 11, 11, 11, 11, 11),
)
flexmock(GithubService)
github_service = GithubService()
flexmock(
GithubService,
get_project=lambda repo, namespace: GithubProject(
"also-not", github_service, "set", github_repo=flexmock()),
)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": DOWNSTREAM_PROJECT_URL},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
pr_create=mocked_pr_create,
)
flexmock(
GithubProject,
get_git_urls=lambda: {"git": UPSTREAM_PROJECT_URL},
fork_create=lambda: None,
)
flexmock(PagureUser, get_username=lambda: "packito")
mock_spec_download_remote_s(distgit)
dglp = LocalProject(
working_dir=str(distgit),
git_url="https://packit.dev/rpms/beer",
git_service=PagureService(),
)
flexmock(
DistGit,
push_to_fork=lambda *args, **kwargs: None,
# let's not hammer the production lookaside cache webserver
is_archive_in_lookaside_cache=lambda archive_path: False,
local_project=dglp,
)
def mocked_new_sources(sources=None):
if not Path(sources).is_file():
raise RuntimeError("archive does not exist")
flexmock(FedPKG,
init_ticket=lambda x=None: None,
new_sources=mocked_new_sources)
pc = get_local_package_config(str(upstream))
pc.dist_git_clone_path = str(distgit)
pc.upstream_project_url = str(upstream)
return upstream, distgit
def lp(self):
if not self._lp:
self._lp = LocalProject(git_project=self.project)
return self._lp
def update_source_git(
config: Config,
source_git: str,
dist_git: str,
revision_range: str,
force: bool,
):
"""Update a source-git repository based on a dist-git repository.
Update a source-git repository with the selected checkout of a spec file
and additional packaging files from a dist-git repository.
Revision range represents part of dist-git history which is supposed
to be synchronized. Use `HEAD~..` if you want to synchronize the last
commit from dist-git. For more information on possible revision range
formats, see gitrevisions(7). If the revision range is not specified,
dist-git commits with no counterpart in source-git will be synchronized.
If patches or the sources file in the spec file changed, the command
exits with return code 2. Such changes are not supported by this
command, code changes should happen in the source-git repo.
Inapplicable changes to the .gitignore file are ignored since the
file may not be synchronized between dist-git and source-git.
This command, by default, performs only local operations and uses the
content of the source-git and dist-git repositories as it is, no checkout
or fetch is performed.
After the synchronization is done, packit will inform about the changes
it has performed and about differences between source-git and dist-git
prior to the synchronization process.
Dist-git commit messages are preserved and used when creating new
source-git commits, but a 'From-dist-git-commit' trailer is appended
to them to mark the hash of the dist-git commit from which they
are created.
Examples
Take the extra (not synchronized) commit(s) of systemd dist-git repo and
copy the spec file and other packaging files into the source-git repo:
\b
$ packit source-git update-source-git rpms/systemd src/systemd
Synchronize changes from the last three dist-git commits:
\b
$ packit source-git update-source-git rpms/systemd src/systemd HEAD~3..
"""
if force and not revision_range:
raise click.BadOptionUsage(
"-f", "revision-range has to be specified when -f/--force is used"
)
source_git_path = pathlib.Path(source_git).resolve()
dist_git_path = pathlib.Path(dist_git).resolve()
package_config = get_local_package_config(
source_git_path, package_config_path=config.package_config_path
)
api = PackitAPI(
config=config,
package_config=package_config,
upstream_local_project=LocalProject(working_dir=source_git_path, offline=True),
downstream_local_project=LocalProject(working_dir=dist_git_path, offline=True),
)
api.update_source_git(
revision_range=revision_range,
check_sync_status=not force,
)
def local_project(self):
""" return an instance of LocalProject """
if self._local_project is None:
self._local_project = LocalProject(path_or_url=self.upstream_project_url)
return self._local_project
def test_local_project_full_name():
project = LocalProject(full_name="namespace/repository_name")
assert project.repo_name == "repository_name"
assert project.namespace == "namespace"
assert project.full_name == "namespace/repository_name"
#!/usr/bin/python3
from packit.local_project import LocalProject
assert LocalProject(full_name="namespace/repository_name")
print("Success", LocalProject)
def sync(
self,
target_url: str,
target_ref: str,
full_name: str,
top_commit: str,
pr_id: int,
pr_url: str,
title: str,
package_config: PackageConfig,
repo_directory: str = None,
):
"""
synchronize selected source-git pull request to respective downstream dist-git repo via a pagure pull request
:param target_url:
:param target_ref:
:param full_name: str, name of the github repo (e.g. user-cont/source-git)
:param top_commit: str, commit hash of the top commit in source-git PR
:param pr_id:
:param pr_url:
:param title:
:param package_config: PackageConfig, configuration of the sg - dg mapping
:param repo_directory: use this directory instead of pulling the url
:return:
"""
logger.info("starting sync for project %s", target_url)
sourcegit = LocalProject(git_url=target_url,
working_dir=repo_directory,
full_name=full_name)
distgit = LocalProject(
git_url=package_config.metadata["dist_git_url"],
branch=f"source-git-{pr_id}",
git_service=PagureService(token=self.pagure_fork_token),
namespace="rpms",
repo_name=package_config.metadata["package_name"],
)
checkout_pr(repo=sourcegit.git_repo, pr_id=pr_id)
with Transformator(sourcegit=sourcegit,
distgit=distgit,
package_config=package_config) as transformator:
transformator.create_archive()
transformator.copy_synced_content_to_distgit_directory(
synced_files=package_config.synced_files)
transformator.add_patches_to_specfile()
commits = transformator.get_commits_to_upstream(
upstream=target_ref)
commits_nice_str = commits_to_nice_str(commits)
logger.debug(f"Commits in source-git PR:\n{commits_nice_str}")
msg = f"upstream commit: {top_commit}\n\nupstream repo: {target_url}"
transformator.commit_distgit(title=title, msg=msg)
project_fork = distgit.git_project.get_fork()
if not project_fork:
logger.info("Creating a fork.")
distgit.git_project.fork_create()
project_fork = distgit.git_project.get_fork()
transformator.push_to_distgit_fork(project_fork=project_fork,
branch_name=distgit.branch)
transformator.reset_checks(
full_name,
pr_id,
github_token=self.github_token,
pagure_user_token=self.pagure_user_token,
)
transformator.update_or_create_dist_git_pr(
distgit.git_project,
pr_id,
pr_url,
top_commit,
title,
source_ref=distgit.branch,
pagure_fork_token=self.pagure_fork_token,
pagure_package_token=self.pagure_package_token,
)
def handle_pull_request(self):
if not self.job.metadata.get("targets"):
logger.error(
"'targets' value is required in packit config for copr_build job"
)
pr_id_int = nested_get(self.event, "number")
pr_id = str(pr_id_int)
self.local_project = LocalProject(
git_project=self.project,
pr_id=pr_id,
git_service=self.project.service,
working_dir=self.config.command_handler_work_dir,
)
self.api = PackitAPI(self.config, self.package_config, self.local_project)
default_project_name = f"{self.project.namespace}-{self.project.repo}-{pr_id}"
owner = self.job.metadata.get("owner") or "packit"
project = self.job.metadata.get("project") or default_project_name
commit_sha = nested_get(self.event, "pull_request", "head", "sha")
r = BuildStatusReporter(self.project, commit_sha)
try:
build_id, repo_url = self.api.run_copr_build(
owner=owner, project=project, chroots=self.job.metadata.get("targets")
)
except SandcastleTimeoutReached:
msg = "You have reached 10-minute timeout while creating the SRPM."
self.project.pr_comment(pr_id_int, msg)
msg = "Timeout reached while creating a SRPM."
r.report("failure", msg)
return HandlerResults(success=False, details={"msg": msg})
except SandcastleCommandFailed as ex:
max_log_size = 1024 * 16 # is 16KB enough?
if len(ex.output) > max_log_size:
output = "Earlier output was truncated\n\n" + ex.output[-max_log_size:]
else:
output = ex.output
msg = (
"There was an error while creating a SRPM.\n"
"\nOutput:"
"\n```\n"
f"{output}"
"\n```"
f"\nReturn code: {ex.rc}"
)
self.project.pr_comment(pr_id_int, msg)
msg = "Failed to create a SRPM."
r.report("failure", msg)
return HandlerResults(success=False, details={"msg": msg})
except FailedCreateSRPM:
msg = "Failed to create a SRPM."
r.report("failure", msg)
return HandlerResults(success=False, details={"msg": msg})
timeout = 60 * 60 * 2
# TODO: document this and enforce int in config
timeout_config = self.job.metadata.get("timeout")
if timeout_config:
timeout = int(timeout_config)
build_state = self.api.watch_copr_build(build_id, timeout, report_func=r.report)
if build_state == "succeeded":
msg = (
f"Congratulations! The build [has finished]({repo_url})"
" successfully. :champagne:\n\n"
"You can install the built RPMs by following these steps:\n\n"
"* `sudo yum install -y dnf-plugins-core` on RHEL 8\n"
"* `sudo dnf install -y dnf-plugins-core` on Fedora\n"
f"* `dnf copr enable {owner}/{project}`\n"
"* And now you can install the packages.\n"
"\nPlease note that the RPMs should be used only in a testing environment."
)
self.project.pr_comment(pr_id_int, msg)
return HandlerResults(success=True, details={})
