def mock_remote_functionality(distgit: Path, upstream: Path):
def mocked_create_pr(*args, **kwargs):
return PullRequestReadOnly(
title="",
id=42,
status=PRStatus.open,
url="",
description="",
author="",
source_branch="",
target_branch="",
created=datetime.datetime(1969, 11, 11, 11, 11, 11, 11),
)
flexmock(GithubService)
github_service = GithubService()
flexmock(
GithubService,
get_project=lambda repo, namespace: GithubProject(
"also-not", github_service, "set", github_repo=flexmock()
),
)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": DOWNSTREAM_PROJECT_URL},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
create_pr=mocked_create_pr,
)
flexmock(
GithubProject,
get_git_urls=lambda: {"git": UPSTREAM_PROJECT_URL},
fork_create=lambda: None,
)
flexmock(PagureUser, get_username=lambda: "packito")
dglp = LocalProject(
working_dir=distgit,
git_url="https://packit.dev/rpms/beer",
git_service=PagureService(),
)
flexmock(
DistGit,
push_to_fork=lambda *args, **kwargs: None,
# let's not hammer the production lookaside cache webserver
is_archive_in_lookaside_cache=lambda archive_path: False,
local_project=dglp,
)
flexmock(DistGit).should_receive("existing_pr").and_return(None)
def mocked_new_sources(sources=None):
if not Path(sources).is_file():
raise RuntimeError("archive does not exist")
flexmock(PkgTool, new_sources=mocked_new_sources)
flexmock(PackitAPI, init_kerberos_ticket=lambda: None)
pc = get_local_package_config(str(upstream))
pc.dist_git_clone_path = str(distgit)
pc.upstream_project_url = str(upstream)
return upstream, distgit
def get_pr(namespace: str, repo: str, pr_num: int) -> PullRequest:
service = PagureService(
instance_url=getenv("PAGURE_URL") or "https://git.stg.centos.org",
token=getenv("PAGURE_TOKEN"),
read_only=True,
)
project = service.get_project(namespace=namespace, repo=repo)
return project.get_pr(pr_num)
def process_new_dg_flag(self, msg):
"""
Process flags from the PR and update source git PR with those flags
:param msg:
:return:
"""
raise NotImplementedError(
"The watching of the Fedora CI is not implemented yet."
)
project_name = msg["msg"]["pullrequest"]["project"]["name"]
logger.info("new flag for PR for %s", project_name)
try:
source_git = get_package_mapping()[project_name]["source-git"]
except KeyError:
logger.info("source git not found")
return
ps = PagureService(token=self.pagure_token)
project = ps.get_project(repo=project_name, namespace="rpms")
pr_id = msg["msg"]["pullrequest"]["id"]
# find info for the matching source git pr
sg_pr_id = project.get_sg_pr_id(pr_id)
# check the commit which tests were running for
commit = project.get_sg_top_commit(pr_id)
if not (sg_pr_id and commit):
logger.info("this doesn't seem to be a source-git related event")
return
repo = self.gh.get_repo(source_git)
sg_pull = repo.get_pull(sg_pr_id)
for c in sg_pull.get_commits():
if c.sha == commit:
gh_commit = c
break
else:
raise RuntimeError("commit was not found in source git")
# Pagure states match github states, coolzies
# https://developer.github.com/v3/repos/statuses/#create-a-status
gh_commit.create_status(
msg["msg"]["flag"]["status"],
target_url=msg["msg"]["flag"]["url"],
description=msg["msg"]["flag"]["comment"],
context=msg["msg"]["flag"]["username"], # simple-koji-ci or Fedora CI
)
def mock_upstream_remote_functionality(upstream_n_distgit):
u, d = upstream_n_distgit
def mocked_pr_create(*args, **kwargs):
return PullRequest(
title="",
id=42,
status=PRStatus.open,
url="",
description="",
author="",
source_branch="",
target_branch="",
created=datetime.datetime(1969, 11, 11, 11, 11, 11, 11),
)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": str(d)},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
pr_create=mocked_pr_create,
)
def mock_download_remote_sources():
""" mock download of the remote archive and place it into dist-git repo """
tarball_path = d / TARBALL_NAME
hops_filename = "hops"
hops_path = d / hops_filename
hops_path.write_text("Cascade\n")
subprocess.check_call(
["tar", "-cf", str(tarball_path), hops_filename], cwd=d)
flexmock(SpecFile, download_remote_sources=mock_download_remote_sources)
flexmock(GithubService, get_project=lambda repo, namespace: flexmock())
flexmock(
DistGit,
push_to_fork=lambda *args, **kwargs: None,
# let's not hammer the production lookaside cache webserver
is_archive_in_lookaside_cache=lambda archive_path: False,
build=lambda scratch: None,
)
def mocked_new_sources(sources=None):
if not Path(sources).is_file():
raise RuntimeError("archive does not exist")
flexmock(FedPKG,
init_ticket=lambda x=None: None,
new_sources=mocked_new_sources)
pc = get_local_package_config(str(u))
pc.downstream_project_url = str(d)
pc.upstream_project_url = str(u)
# https://stackoverflow.com/questions/45580215/using-flexmock-on-python-modules
flexmock(sys.modules["packit.bot_api"]).should_receive(
"get_packit_config_from_repo").and_return(pc)
return u, d
def create_ogr_service(
service_type: str,
service_url: Optional[str] = None,
token: Optional[str] = None,
github_app_id: Optional[str] = None,
github_private_key_path: Optional[str] = None,
):
"""Create a new OGR service for interacting with remote GitForges."""
service_type = service_type.upper()
if service_type == "GITHUB":
ogr_service = GithubService(
token=token,
github_app_id=os.getenv("GITHUB_APP_ID"),
github_private_key_path=os.getenv("GITHUB_PRIVATE_KEY_PATH"),
)
elif service_type == "GITLAB":
ogr_service = GitlabService(token=token, instance_url=service_url)
elif service_type == "PAGURE":
ogr_service = PagureService(
token=token,
instance_url=service_url,
)
else:
raise NotImplementedError(f"Kebechet cannot act on {service_type}")
return ogr_service
def test_get_builds(upstream_n_distgit, expected_results, br_list, number_of_builds):
u, d = upstream_n_distgit
from bodhi.client.bindings import BodhiClient
c = get_test_config()
pc = get_local_package_config(str(u))
pc.downstream_project_url = str(d)
pc.upstream_project_url = str(u)
dg = DistGit(c, pc)
pc = get_local_package_config(str(u))
flexmock(BodhiClient).should_receive("latest_builds").and_return(
BODHI_LATEST_BUILDS
)
status = Status(c, pc, u, dg)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": "foo.git"},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
get_branches=br_list,
)
assert status
table = status.get_builds(number_of_builds)
assert table
assert len(table.keys()) == number_of_builds
assert table == expected_results
def local_project(self):
""" return an instance of LocalProject """
if self._local_project is None:
pagure_service = PagureService(
token=self.pagure_user_token,
instance_url=self.package_config.dist_git_base_url,
read_only=self.config.dry_run,
)
if self.package_config.dist_git_clone_path:
self._local_project = LocalProject(
working_dir=self.package_config.dist_git_clone_path,
git_url=self.package_config.dist_git_package_url,
namespace=self.package_config.dist_git_namespace,
repo_name=self.package_config.downstream_package_name,
git_service=pagure_service,
)
else:
tmpdir = tempfile.mkdtemp(prefix="packit-dist-git")
f = FedPKG(self.fas_user, tmpdir)
f.clone(
self.package_config.downstream_package_name,
tmpdir,
anonymous=not cccolutils.has_creds(),
)
self._local_project = LocalProject(
working_dir=tmpdir,
git_url=self.package_config.dist_git_package_url,
namespace=self.package_config.dist_git_namespace,
repo_name=self.package_config.downstream_package_name,
git_service=pagure_service,
)
self._local_project.working_dir_temporary = True
return self._local_project
def pagure_service(self):
if self._pagure_service is None:
self._pagure_service = PagureService(
token=self.config.pagure_user_token,
read_only=self.config.dry_run,
# TODO: how do we change to stg here? ideally in self.config
)
return self._pagure_service
def local_project(self):
""" return an instance of LocalProject """
if self._local_project is None:
self._local_project = LocalProject(
git_url=self.dist_git_url,
namespace=self.dist_git_namespace,
repo_name=self.package_name,
path_or_url=self.package_config.downstream_project_url,
git_service=PagureService(token=self.pagure_user_token),
)
return self._local_project
def test_basic_distgit_workflow(tmpdir):
pagure_token = os.getenv("PAGURE_TOKEN")
pag = PagureService(
token=pagure_token,
repo="tmux-top",
namespace="rpms",
instance_url="https://src.stg.fedoraproject.org/",
)
print(pag.pagure.whoami())
proj = pag.get_project()
proj.fork_create()
fork = proj.get_fork()
clone_url = fork.get_git_urls()
t = Path(tmpdir)
repo = t.joinpath("repo")
branch_name = "cookie"
subprocess.check_call(["git", "clone", clone_url, repo])
git_set_user_email(repo)
subprocess.check_call(["git", "checkout", "-B", branch_name], cwd=repo)
repo.joinpath("README").write_text("just trying something out\n")
subprocess.check_call(["git", "add", "README"], cwd=repo)
subprocess.check_call(["git", "commit", "-m", "test commit"], cwd=repo)
subprocess.check_call(
["git", "push", "origin", f"{branch_name}:{branch_name}"], cwd=repo)
pr = fork.pr_create("testing PR", "serious description", "master",
branch_name)
proj.pr_comment(pr.id, "howdy!")
def ogr_service(cls) -> BaseGitService:
if cls.SERVICE_NAME == "GITHUB":
return GithubService(
token=cls.AUTH_TOKEN,
github_app_id=cls.GITHUB_APP_ID,
github_app_private_key_path=cls.GITHUB_APP_PRIVATE_KEY_PATH,
)
elif cls.SERVICE_NAME == "GITLAB":
return GitlabService(token=cls.AUTH_TOKEN,
instance_url=cls.SERVICE_URL)
elif cls.SERVICE_NAME == "PAGURE":
return PagureService(token=cls.AUTH_TOKEN,
instance_url=cls.SERVICE_URL)
else:
raise NotImplementedError(
f"Varangian cannot run on {cls.SERVICE_NAME} git services.")
def mock_downstream_remote_functionality(downstream_n_distgit):
u, d = downstream_n_distgit
dglp = LocalProject(
working_dir=d,
git_url="https://packit.dev/rpms/beer",
git_service=PagureService(),
)
flexmock(DistGit, update_branch=lambda *args, **kwargs: "0.0.0", local_project=dglp)
mock_spec_download_remote_s(d)
pc = get_local_package_config(str(u))
pc.dist_git_clone_path = str(d)
pc.upstream_project_url = str(u)
return u, d
def _ogr_service_from_dict(service_dict: Dict[str, str]) -> BaseGitService:
if service_dict["service_name"] == "GITHUB":
return GithubService(
token=service_dict.get("auth_token"),
github_app_id=service_dict.get("github_app_id"),
github_app_private_key_path=service_dict.get(
"github_app_private_key_path"),
)
elif service_dict["service_name"] == "GITLAB":
return GitlabService(token=service_dict.get("auth_token"),
instance_url=service_dict.get("service_url"))
elif service_dict["service_name"] == "PAGURE":
return PagureService(token=service_dict.get("auth_token"),
instance_url=service_dict.get("service_url"))
else:
raise NotImplementedError(
f"Varangian cannot run on {service_dict['service_name']} git services."
)
def test_downstream_pr(upstream_n_distgit, pr_list, number_prs):
u, d = upstream_n_distgit
c = get_test_config()
pc = get_local_package_config(str(u))
pc.downstream_project_url = str(d)
pc.upstream_project_url = str(u)
dg = DistGit(c, pc)
pc = get_local_package_config(str(u))
status = Status(c, pc, u, dg)
flexmock(
PagureProject,
get_git_urls=lambda: {"git": "foo.git"},
fork_create=lambda: None,
get_fork=lambda: PagureProject("", "", PagureService()),
get_pr_list=pr_list,
)
assert status
table = status.get_downstream_prs(number_prs)
assert table
assert len(table) == number_prs
def pagure_service():
return PagureService(token="12345", instance_url="https://pagure.pagure")
def _pagure_service(self):
return PagureService(token=self.config.pagure_user_token)
def pagure_service(pagure_token):
return PagureService(token=pagure_token)
self.logger.info(
f"Added attachment {attachment_id} to bug #{bzid}")
return attachment_id
# PAGURE_TOKEN="ABC" BUGZILLA_API_KEY="XYZ" python3 psbugzilla.py
if __name__ == "__main__":
from os import getenv
from ogr.services.pagure import PagureService
namespace = getenv("NAMESPACE") or "source-git"
repo = getenv("REPO") or "rpm"
pr_id = int(getenv("PR_ID")) or 19
service = PagureService(
instance_url="https://git.stg.centos.org",
token=getenv("PAGURE_TOKEN"),
read_only=True,
)
project = service.get_project(namespace=namespace, repo=repo)
pr = project.get_pr(pr_id)
bz = Bugzilla(url="https://partner-bugzilla.redhat.com",
api_key=getenv("BUGZILLA_API_KEY"))
logging.basicConfig()
bz.logger.setLevel(logging.DEBUG)
description = f"Based on approved CentOS Stream Pull Request: {pr.url}"
bzid, url = bz.create_bug(
product="Red Hat Enterprise Linux 8",
version="CentOS-Stream",
component=repo,
summary=pr.title,
import os
import shutil
from logging import getLogger
from pathlib import Path
from shutil import copyfile
import git
import requests
from ogr.services.pagure import PagureService
logger = getLogger(__name__)
work_dir = Path("/tmp/playground")
readme_path = Path(__file__).parent / "README.md"
service = PagureService(token=os.getenv("PAGURE_TOKEN"),
instance_url="https://git.stg.centos.org/")
class AddMasterBranch:
def __init__(self, pkg_name):
self.pkg_name = pkg_name
self.project = service.get_project(namespace="source-git",
repo=self.pkg_name)
self.pkg_dir = work_dir / self.pkg_name
def run(self):
logger.info(f"Processing package: {self.pkg_name}")
if "master" in self.project.get_branches():
logger.info("\tBranch already exists")
else:
logger.info("\tCreating master branch")
def sync(
self,
target_url: str,
target_ref: str,
full_name: str,
top_commit: str,
pr_id: int,
pr_url: str,
title: str,
package_config: PackageConfig,
repo_directory: str = None,
):
"""
synchronize selected source-git pull request to respective downstream dist-git repo via a pagure pull request
:param target_url:
:param target_ref:
:param full_name: str, name of the github repo (e.g. user-cont/source-git)
:param top_commit: str, commit hash of the top commit in source-git PR
:param pr_id:
:param pr_url:
:param title:
:param package_config: PackageConfig, configuration of the sg - dg mapping
:param repo_directory: use this directory instead of pulling the url
:return:
"""
logger.info("starting sync for project %s", target_url)
sourcegit = LocalProject(git_url=target_url,
working_dir=repo_directory,
full_name=full_name)
distgit = LocalProject(
git_url=package_config.metadata["dist_git_url"],
branch=f"source-git-{pr_id}",
git_service=PagureService(token=self.pagure_fork_token),
namespace="rpms",
repo_name=package_config.metadata["package_name"],
)
checkout_pr(repo=sourcegit.git_repo, pr_id=pr_id)
with Transformator(sourcegit=sourcegit,
distgit=distgit,
package_config=package_config) as transformator:
transformator.create_archive()
transformator.copy_synced_content_to_distgit_directory(
synced_files=package_config.synced_files)
transformator.add_patches_to_specfile()
commits = transformator.get_commits_to_upstream(
upstream=target_ref)
commits_nice_str = commits_to_nice_str(commits)
logger.debug(f"Commits in source-git PR:\n{commits_nice_str}")
msg = f"upstream commit: {top_commit}\n\nupstream repo: {target_url}"
transformator.commit_distgit(title=title, msg=msg)
project_fork = distgit.git_project.get_fork()
if not project_fork:
logger.info("Creating a fork.")
distgit.git_project.fork_create()
project_fork = distgit.git_project.get_fork()
transformator.push_to_distgit_fork(project_fork=project_fork,
branch_name=distgit.branch)
transformator.reset_checks(
full_name,
pr_id,
github_token=self.github_token,
pagure_user_token=self.pagure_user_token,
)
transformator.update_or_create_dist_git_pr(
distgit.git_project,
pr_id,
pr_url,
top_commit,
title,
source_ref=distgit.branch,
pagure_fork_token=self.pagure_fork_token,
pagure_package_token=self.pagure_package_token,
)
