def test_get_builds(upstream_n_distgit, expected_results, br_list, number_of_builds):
u, d = upstream_n_distgit
from bodhi.client.bindings import BodhiClient
c = get_test_config()
pc = get_local_package_config(str(u))
pc.downstream_project_url = str(d)
pc.upstream_project_url = str(u)
dg = DistGit(c, pc)
pc = get_local_package_config(str(u))
flexmock(BodhiClient).should_receive("latest_builds").and_return(
BODHI_LATEST_BUILDS
)
status = Status(c, pc, u, dg)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": "foo.git"},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
get_branches=br_list,
)
assert status
table = status.get_builds(number_of_builds)
assert table
assert len(table.keys()) == number_of_builds
assert table == expected_results
def mock_remote_functionality(distgit: Path, upstream: Path):
def mocked_create_pr(*args, **kwargs):
return PullRequestReadOnly(
title="",
id=42,
status=PRStatus.open,
url="",
description="",
author="",
source_branch="",
target_branch="",
created=datetime.datetime(1969, 11, 11, 11, 11, 11, 11),
)
flexmock(GithubService)
github_service = GithubService()
flexmock(
GithubService,
get_project=lambda repo, namespace: GithubProject(
"also-not", github_service, "set", github_repo=flexmock()
),
)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": DOWNSTREAM_PROJECT_URL},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
create_pr=mocked_create_pr,
)
flexmock(
GithubProject,
get_git_urls=lambda: {"git": UPSTREAM_PROJECT_URL},
fork_create=lambda: None,
)
flexmock(PagureUser, get_username=lambda: "packito")
dglp = LocalProject(
working_dir=distgit,
git_url="https://packit.dev/rpms/beer",
git_service=PagureService(),
)
flexmock(
DistGit,
push_to_fork=lambda *args, **kwargs: None,
# let's not hammer the production lookaside cache webserver
is_archive_in_lookaside_cache=lambda archive_path: False,
local_project=dglp,
)
flexmock(DistGit).should_receive("existing_pr").and_return(None)
def mocked_new_sources(sources=None):
if not Path(sources).is_file():
raise RuntimeError("archive does not exist")
flexmock(PkgTool, new_sources=mocked_new_sources)
flexmock(PackitAPI, init_kerberos_ticket=lambda: None)
pc = get_local_package_config(str(upstream))
pc.dist_git_clone_path = str(distgit)
pc.upstream_project_url = str(upstream)
return upstream, distgit
def test_set_spec_ver_empty_changelog(tmpdir):
t = Path(str(tmpdir))
u_remote_path = t / "upstream_remote"
u_remote_path.mkdir(parents=True, exist_ok=True)
subprocess.check_call(["git", "init", "--bare", "."], cwd=u_remote_path)
u = t / "upstream_git"
shutil.copytree(EMPTY_CHANGELOG, u)
initiate_git_repo(u, tag="0.1.0")
with cwd(tmpdir):
c = get_test_config()
pc = get_local_package_config(str(u))
pc.upstream_project_url = str(u)
lp = LocalProject(working_dir=str(u))
ups = Upstream(c, pc, lp)
new_ver = "1.2.3"
ups.specfile.set_spec_version(version=new_ver, changelog_entry="- asdqwe")
assert ups.get_specfile_version() == new_ver
assert "%changelog" not in u.joinpath("beer.spec").read_text()
def test_basic_local_update_direct_push(upstream_distgit_remote,
mock_remote_functionality_upstream):
""" basic propose-update test: mock remote API, use local upstream and dist-git """
upstream, distgit, remote_dir = upstream_distgit_remote
with cwd(upstream):
c = get_test_config()
pc = get_local_package_config(str(upstream))
pc.upstream_project_url = str(upstream)
pc.dist_git_clone_path = str(distgit)
up_lp = LocalProject(working_dir=str(upstream))
api = PackitAPI(c, pc, up_lp)
api.sync_release("master", "0.1.0", create_pr=False)
remote_dir_clone = Path(f"{remote_dir}-clone")
subprocess.check_call(
["git", "clone", remote_dir,
str(remote_dir_clone)],
cwd=str(remote_dir_clone.parent),
)
spec = get_specfile(str(remote_dir_clone / "beer.spec"))
assert spec.get_version() == "0.1.0"
assert (remote_dir_clone / "README.packit").is_file()
def test_set_spec_ver_empty_changelog(tmp_path):
u_remote_path = tmp_path / "upstream_remote"
u_remote_path.mkdir(parents=True, exist_ok=True)
create_new_repo(u_remote_path, ["--bare"])
u = tmp_path / "upstream_git"
shutil.copytree(EMPTY_CHANGELOG, u)
initiate_git_repo(u, tag="0.1.0")
with cwd(tmp_path):
c = get_test_config()
pc = get_local_package_config(str(u))
pc.upstream_project_url = str(u)
lp = LocalProject(working_dir=u)
ups = Upstream(c, pc, lp)
new_ver = "1.2.3"
ups.specfile.version = new_ver
ups.specfile.add_changelog_entry("- asdqwe")
assert ups.get_specfile_version() == new_ver
assert "%changelog" not in u.joinpath("beer.spec").read_text()
def get_packit_api(config: Config,
local_project: LocalProject,
dist_git_path: str = None):
"""
Load the package config, set other options and return the PackitAPI
"""
package_config = get_local_package_config(local_project.working_dir,
try_local_dir_last=True)
if dist_git_path:
package_config.dist_git_clone_path = dist_git_path
if dist_git_path and dist_git_path == local_project.working_dir:
PackitAPI(
config=config,
package_config=package_config,
upstream_local_project=None,
downstream_local_project=local_project,
)
remote_urls: List[str] = []
for remote in local_project.git_repo.remotes:
remote_urls += remote.urls
upstream_hostname = (get_hostname_or_none(
url=package_config.upstream_project_url)
if package_config.upstream_project_url else None)
lp_upstream = None
lp_downstream = None
for url in remote_urls:
remote_hostname = get_hostname_or_none(url=url)
if not remote_hostname:
continue
if upstream_hostname:
if remote_hostname == upstream_hostname:
lp_upstream = local_project
logger.info("Input directory is an upstream repository.")
break
if package_config.dist_git_base_url and (
remote_hostname in package_config.dist_git_base_url
or remote_hostname in DIST_GIT_HOSTNAME_CANDIDATES):
lp_downstream = local_project
logger.info("Input directory is a downstream repository.")
break
else:
lp_upstream = local_project
# fallback, this is the past behavior
logger.info("Input directory is an upstream repository.")
api = PackitAPI(
config=config,
package_config=package_config,
upstream_local_project=lp_upstream,
downstream_local_project=lp_downstream,
)
return api
def source_git_status(config: Config, source_git: str, dist_git: str):
"""Tell the synchronization status of a source-git and a dist-git repo.
This command checks the commit history in the provided source-git
and dist-git repos and informs about the range of commits to be
synchronized from dist-git to source-git or the other way around,
or informs that the repositories are in sync.
If possible, the status command also provides instructions on how
to synchronize the repositories.
"""
source_git_path = pathlib.Path(source_git).resolve()
dist_git_path = pathlib.Path(dist_git).resolve()
package_config = get_local_package_config(
source_git_path, package_config_path=config.package_config_path)
api = PackitAPI(
config=config,
package_config=package_config,
upstream_local_project=LocalProject(working_dir=source_git_path,
offline=True),
downstream_local_project=LocalProject(working_dir=dist_git_path,
offline=True),
)
click.echo(api.sync_status_string(source_git=source_git,
dist_git=dist_git))
def sync_using_fedmsg_dict(self, fedmsg_dict: dict) -> None:
"""
Sync the pr to the dist-git.
:param fedmsg_dict: dict, fedmsg of a newly opened PR
"""
try:
target_url = fedmsg_dict["msg"]["pull_request"]["base"]["repo"][
"html_url"]
except (KeyError, ValueError) as ex:
logger.debug("ex = %s", ex)
logger.error("invalid fedmsg format")
return
try:
package_config = get_local_package_config()
if package_config:
logger.debug("Using local package config.")
else:
package_config = self.get_packit_config(
repo=fedmsg_dict["msg"]["pull_request"]["head"]["repo"]
["name"],
namespace=fedmsg_dict["msg"]["pull_request"]["head"]
["repo"]["owner"]["login"],
branch=fedmsg_dict["msg"]["pull_request"]["head"]["ref"],
)
except Exception as ex:
logger.info("no source-git mapping for project %s", target_url)
return
try:
msg_id = fedmsg_dict["msg_id"]
except KeyError:
logger.error("provided message is not a fedmsg (missing msg_id)")
return
try:
nice_msg = json.dumps(fedmsg_dict, indent=4)
logger.debug(f"Processing fedmsg:\n{nice_msg}")
self.sync(
target_url=fedmsg_dict["msg"]["pull_request"]["base"]["repo"]
["html_url"],
target_ref=fedmsg_dict["msg"]["pull_request"]["base"]["ref"],
full_name=fedmsg_dict["msg"]["pull_request"]["base"]["repo"]
["full_name"],
top_commit=fedmsg_dict["msg"]["pull_request"]["head"]["sha"],
pr_id=fedmsg_dict["msg"]["pull_request"]["number"],
pr_url=fedmsg_dict["msg"]["pull_request"]["html_url"],
title=fedmsg_dict["msg"]["pull_request"]["title"],
package_config=package_config,
)
except ConnectionError as ex:
# TODO: Retry on connection error
logger.warning(f"Connection error on processing a msg {msg_id}")
logger.debug(str(ex))
return
except Exception as ex:
logger.warning(f"Error on processing a msg {msg_id}")
logger.debug(str(ex))
return
def test_set_spec_macro_source(tmp_path):
u_remote_path = tmp_path / "upstream_remote"
u_remote_path.mkdir(parents=True, exist_ok=True)
create_new_repo(u_remote_path, ["--bare"])
u = tmp_path / "upstream_git"
shutil.copytree(UPSTREAM_MACRO_IN_SOURCE, u)
initiate_git_repo(u, tag="0.1.0")
with cwd(tmp_path):
c = get_test_config()
pc = get_local_package_config(str(u))
pc.upstream_project_url = str(u)
lp = LocalProject(working_dir=u)
ups = Upstream(c, pc, lp)
expected_sources = ups.specfile.sources
new_ver = "1.2.3"
ups.specfile.set_spec_version(version=new_ver, changelog_entry="- asdqwe")
assert ups.get_specfile_version() == new_ver
assert ups.specfile.sources == expected_sources
expected_sources = ups.specfile.sources
new_rel = "121"
ups.specfile.set_spec_version(release=new_rel)
assert ups.specfile.get_release() == new_rel
assert ups.specfile.sources == expected_sources
def mock_upstream_remote_functionality(upstream_n_distgit):
u, d = upstream_n_distgit
def mocked_pr_create(*args, **kwargs):
return PullRequest(
title="",
id=42,
status=PRStatus.open,
url="",
description="",
author="",
source_branch="",
target_branch="",
created=datetime.datetime(1969, 11, 11, 11, 11, 11, 11),
)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": str(d)},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
pr_create=mocked_pr_create,
)
def mock_download_remote_sources():
""" mock download of the remote archive and place it into dist-git repo """
tarball_path = d / TARBALL_NAME
hops_filename = "hops"
hops_path = d / hops_filename
hops_path.write_text("Cascade\n")
subprocess.check_call(
["tar", "-cf", str(tarball_path), hops_filename], cwd=d)
flexmock(SpecFile, download_remote_sources=mock_download_remote_sources)
flexmock(GithubService, get_project=lambda repo, namespace: flexmock())
flexmock(
DistGit,
push_to_fork=lambda *args, **kwargs: None,
# let's not hammer the production lookaside cache webserver
is_archive_in_lookaside_cache=lambda archive_path: False,
build=lambda scratch: None,
)
def mocked_new_sources(sources=None):
if not Path(sources).is_file():
raise RuntimeError("archive does not exist")
flexmock(FedPKG,
init_ticket=lambda x=None: None,
new_sources=mocked_new_sources)
pc = get_local_package_config(str(u))
pc.downstream_project_url = str(d)
pc.upstream_project_url = str(u)
# https://stackoverflow.com/questions/45580215/using-flexmock-on-python-modules
flexmock(sys.modules["packit.bot_api"]).should_receive(
"get_packit_config_from_repo").and_return(pc)
return u, d
def test_update_on_cockpit_ostree_pr_exists(cockpit_ostree):
upstream_path, dist_git_path = cockpit_ostree
def mocked_new_sources(sources=None):
if not Path(sources).is_file():
raise RuntimeError("archive does not exist")
flexmock(PkgTool, new_sources=mocked_new_sources)
flexmock(PackitAPI, init_kerberos_ticket=lambda: None)
flexmock(
DistGit,
push_to_fork=lambda *args, **kwargs: None,
is_archive_in_lookaside_cache=lambda archive_path: False,
upload_to_lookaside_cache=lambda archive, pkg_tool: None,
download_upstream_archive=lambda: "the-archive",
)
pr = flexmock(url="https://example.com/pull/1")
flexmock(DistGit).should_receive("existing_pr").and_return(pr)
pc = get_local_package_config(str(upstream_path))
up_lp = LocalProject(working_dir=upstream_path)
c = get_test_config()
api = PackitAPI(c, pc, up_lp)
api._dg = DistGit(c, pc)
api._dg._local_project = LocalProject(working_dir=dist_git_path)
with cwd(upstream_path):
assert pr == api.sync_release(
dist_git_branch="main",
use_local_content=False,
version="179",
force_new_sources=False,
create_pr=True,
)
def test_update_on_cockpit_ostree(cockpit_ostree):
def mocked_new_sources(sources=None):
if not Path(sources).is_file():
raise RuntimeError("archive does not exist")
flexmock(FedPKG,
init_ticket=lambda x=None: None,
new_sources=mocked_new_sources)
flexmock(
DistGit,
push_to_fork=lambda *args, **kwargs: None,
is_archive_in_lookaside_cache=lambda archive_path: False,
)
flexmock(
PackitAPI,
push_and_create_pr=lambda pr_title, pr_description, dist_git_branch:
None,
)
pc = get_local_package_config(str(cockpit_ostree))
up_lp = LocalProject(working_dir=str(cockpit_ostree))
c = get_test_config()
api = PackitAPI(c, pc, up_lp)
with cwd(cockpit_ostree):
api.sync_release(
"master",
use_local_content=False,
version="179",
force_new_sources=False,
create_pr=True,
)
assert api.dg.download_upstream_archive().is_file()
def downstream_config(self) -> Optional[PackageConfig]:
if not self._downstream_config:
try:
self._downstream_config = get_local_package_config(
self.local_project.working_dir)
except PackitConfigException:
return None
return self._downstream_config
def test_get_updates(upstream_n_distgit, expected_status, number_of_updates):
u, d = upstream_n_distgit
from bodhi.client.bindings import BodhiClient
c = get_test_config()
pc = get_local_package_config(str(u))
pc.downstream_project_url = str(d)
pc.upstream_project_url = str(u)
dg = DistGit(c, pc)
pc = get_local_package_config(str(u))
flexmock(BodhiClient).should_receive("query").and_return(BODHI_UPDATES)
status = Status(c, pc, u, dg)
assert status
table = status.get_updates(number_of_updates=number_of_updates)
assert table
assert len(table) == number_of_updates
assert table == expected_status
def distgit_instance(upstream_n_distgit, mock_remote_functionality_upstream):
u, d = upstream_n_distgit
c = get_test_config()
pc = get_local_package_config(str(u))
pc.downstream_project_url = str(d)
pc.upstream_project_url = str(u)
dg = DistGit(c, pc)
return d, dg
def test_srpm_snapd(snapd):
pc = get_local_package_config(str(snapd))
up_lp = LocalProject(working_dir=str(snapd))
c = get_test_config()
api = PackitAPI(c, pc, up_lp)
with cwd(snapd):
path = api.create_srpm()
assert path.exists()
build_srpm(path)
def update(config, dist_git_path, upstream_git_path, dist_git_branch, repo):
"""
Release current upstream release into Fedora
"""
package_config = get_local_package_config(directory=repo.working_dir)
package_config.downstream_project_url = dist_git_path
package_config.upstream_project_url = upstream_git_path
api = PackitAPI(config=config, package_config=package_config)
api.sync_release(dist_git_branch)
def mock_api_for_source_git(
sourcegit: Path, distgit: Path, up_local_project: LocalProject
):
with cwd(sourcegit):
c = get_test_config()
pc = get_local_package_config(str(sourcegit))
pc.upstream_project_url = str(sourcegit)
pc.dist_git_clone_path = str(distgit)
return PackitAPI(c, pc, up_local_project)
def sg2dg(config, dest_dir, no_new_sources, upstream_ref, repo, version):
"""
Convert source-git repo to dist-git repo.
1. Create tarball from the source git repo.
2. Create patches from the downstream commits.
3. Copy the redhat/ dir to the dist-git.
4. Take the tarball and upload it to lookaside cache.
5. The output is the directory (= dirty git repo)
"""
package_config = get_local_package_config()
sourcegit = LocalProject(git_url=repo)
if not package_config:
package_config = get_local_package_config(directory=sourcegit.working_dir)
distgit = LocalProject(
git_url=package_config.metadata["dist_git_url"],
namespace="rpms",
repo_name=package_config.metadata["package_name"],
working_dir=dest_dir,
)
with Transformator(
sourcegit=sourcegit,
distgit=distgit,
version=version,
fas_username=config.fas_user,
package_config=package_config,
) as t:
t.create_archive()
t.copy_synced_content_to_distgit_directory(
synced_files=package_config.synced_files
)
patches = t.create_patches(upstream=upstream_ref)
t.add_patches_to_specfile(patch_list=patches)
if not no_new_sources:
t.upload_archive_to_lookaside_cache(config.keytab)
else:
logger.debug("Skipping fedpkg new-sources.")
click.echo(f"{distgit.working_dir}")
def update_source_git(
config: Config, source_git: str, dist_git: str, revision_range: str
):
"""Update a source-git repository based on a dist-git repository.
Update a source-git repository with the selected checkout of a spec file
and additional packaging files from a dist-git repository.
Revision range represents part of dist-git history which is supposed
to be synchronized. Use `HEAD~..` if you want to synchronize the last
commit from dist-git. For more information on possible revision range
formats, see gitrevisions(7).
If patches or the sources file in the spec file changed, the command
exits with return code 2. Such changes are not supported by this
command, code changes should happen in the source-git repo.
Inapplicable changes to the .gitignore file are ignored since the
file may not be synchronized between dist-git and source-git.
This command, by default, performs only local operations and uses the
content of the source-git and dist-git repositories as it is, no checkout
or fetch is performed.
After the synchronization is done, packit will inform about the changes
it has performed and about differences between source-git and dist-git
prior to the synchronization process.
Dist-git commit messages are preserved and used when creating new
source-git commits.
Examples
Take the last commit (HEAD) of systemd dist-git repo and copy the
spec file and other packaging files into the source-git repo:
\b
$ packit source-git update-source-git rpms/systemd src/systemd HEAD~..
Synchronize changes from the last three dist-git commits:
\b
$ packit source-git update-source-git rpms/systemd src/systemd HEAD~3..
"""
source_git_path = pathlib.Path(source_git).resolve()
dist_git_path = pathlib.Path(dist_git).resolve()
package_config = get_local_package_config(
source_git_path, package_config_path=config.package_config_path
)
api = PackitAPI(
config=config,
package_config=package_config,
upstream_local_project=LocalProject(working_dir=source_git_path, offline=True),
downstream_local_project=LocalProject(working_dir=dist_git_path, offline=True),
)
api.update_source_git(revision_range=revision_range)
def sg2srpm(config, dest_dir, upstream_ref, version, repo):
"""
Generate a srpm from packit.
This script is meant to accept a source git repo with a branch as an input and produce a SRPM.
It is expected to do this:
1. clone the repo
2. create archive out of the sources
3. create SRPM
"""
package_config = get_local_package_config()
sourcegit = LocalProject(git_url=repo)
if not package_config:
package_config = get_local_package_config(directory=sourcegit.working_dir)
distgit = LocalProject(
git_url=package_config.metadata["dist_git_url"],
namespace="rpms",
repo_name=package_config.metadata["package_name"],
working_dir=dest_dir,
)
distgit.working_dir_temporary = False
with Transformator(
sourcegit=sourcegit,
distgit=distgit,
version=version,
fas_username=config.fas_user,
package_config=package_config,
) as t:
t.download_upstream_archive()
t.copy_synced_content_to_distgit_directory(
synced_files=package_config.synced_files
)
patches = t.create_patches(upstream=upstream_ref)
t.add_patches_to_specfile(patch_list=patches)
srpm = t.create_srpm()
click.echo(srpm)
def distgit_instance(upstream_and_remote, distgit_and_remote,
mock_remote_functionality_upstream):
u, _ = upstream_and_remote
d, _ = distgit_and_remote
c = get_test_config()
pc = get_local_package_config(str(u))
pc.dist_git_clone_path = str(d)
pc.upstream_project_url = str(u)
dg = DistGit(c, pc)
return d, dg
def test_srpm_on_cockpit_ostree(cockpit_ostree):
upstream_path, dist_git_path = cockpit_ostree
pc = get_local_package_config(str(upstream_path))
up_lp = LocalProject(working_dir=str(upstream_path))
c = get_test_config()
api = PackitAPI(c, pc, up_lp)
with cwd(upstream_path):
api.create_srpm()
def api_instance_source_git(sourcegit_n_distgit):
sourcegit, distgit = sourcegit_n_distgit
with cwd(sourcegit):
c = get_test_config()
pc = get_local_package_config(str(sourcegit))
pc.upstream_project_url = str(sourcegit)
pc.downstream_project_url = str(distgit)
up_lp = LocalProject(path_or_url=str(sourcegit))
api = PackitAPI(c, pc, up_lp)
return api
def api_instance_source_git(sourcegit_and_remote, distgit_and_remote):
sourcegit, _ = sourcegit_and_remote
distgit, _ = distgit_and_remote
with cwd(sourcegit):
c = get_test_config()
pc = get_local_package_config(str(sourcegit))
pc.upstream_project_url = str(sourcegit)
pc.dist_git_clone_path = str(distgit)
up_lp = LocalProject(working_dir=str(sourcegit))
api = PackitAPI(c, pc, up_lp)
return api
def api_instance(upstream_n_distgit):
u, d = upstream_n_distgit
chdir(u)
c = get_test_config()
pc = get_local_package_config(str(u))
pc.upstream_project_url = str(u)
api = PackitAPI(c, pc)
return u, d, api
def test_downstream_pr(upstream_n_distgit, pr_list, number_prs):
u, d = upstream_n_distgit
c = get_test_config()
pc = get_local_package_config(str(u))
pc.downstream_project_url = str(d)
pc.upstream_project_url = str(u)
dg = DistGit(c, pc)
pc = get_local_package_config(str(u))
status = Status(c, pc, u, dg)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": "foo.git"},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
get_pr_list=pr_list,
)
assert status
table = status.get_downstream_prs(number_prs)
assert table
assert len(table) == number_prs
def upstream_instance(upstream_n_distgit, tmpdir):
with cwd(tmpdir):
u, d = upstream_n_distgit
c = get_test_config()
pc = get_local_package_config(str(u))
pc.upstream_project_url = str(u)
pc.dist_git_clone_path = str(d)
lp = LocalProject(working_dir=str(u))
ups = Upstream(c, pc, lp)
yield u, ups
def upstream_instance(upstream_n_distgit):
u, d = upstream_n_distgit
chdir(u)
c = get_test_config()
pc = get_local_package_config(str(u))
pc.upstream_project_url = str(u)
pc.downstream_project_url = str(d)
ups = Upstream(c, pc)
return u, ups
def test_basic_build(upstream_n_distgit, mock_remote_functionality_upstream):
u, d = upstream_n_distgit
with cwd(u):
c = get_test_config()
pc = get_local_package_config(str(u))
pc.upstream_project_url = str(u)
pc.downstream_project_url = str(d)
up_lp = LocalProject(path_or_url=u)
api = PackitAPI(c, pc, up_lp)
api.build("master")
