def post(self):
"""
A webhook used by Packit-as-a-Service Gitlab hook.
"""
msg = request.json
if not msg:
logger.debug(
"/webhooks/gitlab: we haven't received any JSON data.")
return "We haven't received any JSON data.", HTTPStatus.BAD_REQUEST
if all([msg.get("zen"), msg.get("hook_id"), msg.get("hook")]):
logger.debug(
f"/webhooks/gitlab received ping event: {msg['hook']}")
return "Pong!", HTTPStatus.OK
try:
self.validate_token()
except ValidationFailed as exc:
logger.info(f"/webhooks/gitlab {exc}")
return str(exc), HTTPStatus.UNAUTHORIZED
if not self.interested():
return "Thanks but we don't care about this event", HTTPStatus.ACCEPTED
celery_app.send_task(
name=getenv("CELERY_MAIN_TASK_NAME")
or CELERY_DEFAULT_MAIN_TASK_NAME,
kwargs={"event": msg},
)
return "Webhook accepted. We thank you, Gitlab.", HTTPStatus.ACCEPTED
def post(self):
"""
A webhook used by Packit-as-a-Service GitHub App.
"""
msg = request.json
if not msg:
logger.debug("/webhooks/github: we haven't received any JSON data.")
return "We haven't received any JSON data.", HTTPStatus.BAD_REQUEST
if all([msg.get("zen"), msg.get("hook_id"), msg.get("hook")]):
logger.debug(f"/webhooks/github received ping event: {msg['hook']}")
return "Pong!", HTTPStatus.OK
try:
self.validate_signature()
except ValidationFailed as exc:
logger.info(f"/webhooks/github {exc}")
return str(exc), HTTPStatus.UNAUTHORIZED
if not self.interested():
return "Thanks but we don't care about this event", HTTPStatus.ACCEPTED
# TODO: define task names at one place
celery_app.send_task(
name="task.steve_jobs.process_message", kwargs={"event": msg}
)
return "Webhook accepted. We thank you, Github.", HTTPStatus.ACCEPTED
def post(self):
"""
Submit Testing Farm results
"""
msg = request.json
if not msg:
logger.debug(
"/testing-farm/results: we haven't received any JSON data.")
return "We haven't received any JSON data.", HTTPStatus.BAD_REQUEST
try:
self.validate_testing_farm_request()
except ValidationFailed as exc:
logger.info(f"/testing-farm/results {exc}")
return str(exc), HTTPStatus.UNAUTHORIZED
# There's only one key in the msg,
# so make sure we don't confuse this with something else
msg["source"] = "testing-farm"
celery_app.send_task(
name=getenv("CELERY_MAIN_TASK_NAME")
or CELERY_DEFAULT_MAIN_TASK_NAME,
kwargs={"event": msg},
)
return "Test results accepted", HTTPStatus.OK
def listen_to_fedmsg(message_id):
"""
Listen to events on fedmsg and process them.
if MESSAGE-ID is specified, process only the selected messages
"""
consumerino = Consumerino()
if message_id:
for msg_id in message_id:
fedmsg_dict = consumerino.fetch_fedmsg_dict(msg_id)
logger.debug(f"Processing {fedmsg_dict}")
celery_app.send_task(name="task.steve_jobs.process_message",
kwargs={"event": fedmsg_dict})
else:
for topic, msg in consumerino.yield_all_messages():
if do_we_process_fedmsg_topic(topic):
logger.debug(f"Processing topic {topic}, msg {msg}")
celery_app.send_task(
name="task.steve_jobs.process_message",
kwargs={
"event": msg,
"topic": topic
},
)
def handle_rpm_build_start(
self, build_id: int, web_url: str, waiting_for_srpm: bool = False
):
"""
Create models for Copr build chroots and report start of RPM build
if the SRPM is already built.
"""
unprocessed_chroots = []
for chroot in self.build_targets:
if chroot not in self.available_chroots:
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.error,
description=f"Not supported target: {chroot}",
url=get_srpm_build_info_url(self.srpm_model.id),
chroot=chroot,
)
self.monitor_not_submitted_copr_builds(1, "not_supported_target")
unprocessed_chroots.append(chroot)
continue
copr_build = CoprBuildModel.create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
project_name=self.job_project,
owner=self.job_owner,
web_url=web_url,
target=chroot,
status="waiting_for_srpm" if waiting_for_srpm else "pending",
run_model=self.run_model,
task_accepted_time=self.metadata.task_accepted_time,
)
if not waiting_for_srpm:
url = get_copr_build_info_url(id_=copr_build.id)
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.running,
description="Starting RPM build...",
url=url,
chroot=chroot,
)
if unprocessed_chroots:
unprocessed = "\n".join(sorted(unprocessed_chroots))
available = "\n".join(sorted(self.available_chroots))
self.status_reporter.comment(
body="There are build targets that are not supported by COPR.\n"
"<details>\n<summary>Unprocessed build targets</summary>\n\n"
f"```\n{unprocessed}\n```\n</details>\n"
"<details>\n<summary>Available build targets</summary>\n\n"
f"```\n{available}\n```\n</details>",
)
# release the hounds!
celery_app.send_task(
"task.babysit_copr_build",
args=(build_id,),
countdown=120, # do the first check in 120s
)
def run_copr_build(self) -> HandlerResults:
if not (self.job_build or self.job_tests):
msg = "No copr_build or tests job defined."
# we can't report it to end-user at this stage
return HandlerResults(success=False, details={"msg": msg})
self.report_status_to_all(description="Building SRPM ...",
state=CommitStatus.pending)
build_metadata = self._run_copr_build_and_save_output()
srpm_build_model = SRPMBuild.create(build_metadata.srpm_logs)
if build_metadata.srpm_failed:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=CommitStatus.failure,
description=msg,
url=get_srpm_log_url(srpm_build_model.id),
)
return HandlerResults(success=False, details={"msg": msg})
for chroot in self.build_chroots:
copr_build = CoprBuild.get_or_create(
pr_id=self.pr_id,
build_id=str(build_metadata.copr_build_id),
commit_sha=self.event.commit_sha,
repo_name=self.project.repo,
namespace=self.project.namespace,
project_name=self.job_project,
owner=self.job_owner,
web_url=build_metadata.copr_web_url,
target=chroot,
status="pending",
srpm_build=srpm_build_model,
)
url = get_log_url(id_=copr_build.id)
self.report_status_to_all_for_chroot(
state=CommitStatus.pending,
description="Building RPM ...",
url=url,
chroot=chroot,
)
self.copr_build_model.build_id = build_metadata.copr_build_id
self.copr_build_model.save()
# release the hounds!
celery_app.send_task(
"task.babysit_copr_build",
args=(build_metadata.copr_build_id, ),
countdown=120, # do the first check in 120s
)
return HandlerResults(success=True, details={})
def testing_farm_results():
msg = request.get_json()
if not msg:
logger.debug(
"/testing-farm/results: we haven't received any JSON data.")
return "We haven't received any JSON data."
if not validate_testing_farm_request():
abort(401) # Unauthorized
celery_app.send_task(name="task.steve_jobs.process_message",
kwargs={"event": msg})
return json.dumps({"status": 200, "message": "Test results accepted"})
def github_webhook():
msg = request.get_json()
if not msg:
logger.debug("/webhooks/github: we haven't received any JSON data.")
return "We haven't received any JSON data."
if all([msg.get("zen"), msg.get("hook_id"), msg.get("hook")]):
logger.debug(f"/webhooks/github received ping event: {msg['hook']}")
return "Pong!"
if not _validate_signature():
abort(401) # Unauthorized
# TODO: define task names at one place
celery_app.send_task(name="task.steve_jobs.process_message", kwargs={"event": msg})
return "Webhook accepted. We thank you, Github."
def post(self):
"""
Submit Testing Farm results
"""
msg = request.json
if not msg:
logger.debug(
"/testing-farm/results: we haven't received any JSON data.")
return "We haven't received any JSON data.", HTTPStatus.BAD_REQUEST
try:
self.validate_testing_farm_request()
except ValidationFailed as exc:
logger.info(f"/testing-farm/results {exc}")
return str(exc), HTTPStatus.UNAUTHORIZED
celery_app.send_task(name="task.steve_jobs.process_message",
kwargs={"event": msg})
return "Test results accepted", HTTPStatus.ACCEPTED
def testing_farm_results():
"""
Expected format:
{
"artifact": {
"commit-sha": "08bfc38f15082bdf9ba964c3bbd04878666d1d56",
"copr-chroot": "fedora-29-x86_64",
"copr-repo-name": "packit/packit-service-hello-world-14",
"git-ref": "08bfc38f15082bdf9ba964c3bbd04878666d1d56",
"git-url": "https://github.com/packit-service/hello-world",
"repo-name": "hello-world",
"repo-namespace": "packit-service"
},
"message": "Command 'git' not found",
"pipeline": {
"id": "614d240a-1e27-4758-ad6a-ed3d34281924"
},
"result": "error",
"token": "HERE-IS-A-VALID-TOKEN",
"url": "https://console-testing-farm.apps.ci.centos.org/pipeline/<ID>"
}
:return: {"status": 200, "message": "Test results accepted"}
"""
msg = request.get_json()
if not msg:
logger.debug(
"/testing-farm/results: we haven't received any JSON data.")
return "We haven't received any JSON data."
if not validate_testing_farm_request():
abort(401) # Unauthorized
celery_app.send_task(name="task.steve_jobs.process_message",
kwargs={"event": msg})
return json.dumps({"status": 200, "message": "Test results accepted"})
def run_copr_build(self) -> HandlerResults:
if not (self.job_build or self.job_tests):
msg = "No copr_build or tests job defined."
# we can't report it to end-user at this stage
return HandlerResults(success=False, details={"msg": msg})
self.report_status_to_all(
description="Building SRPM ...",
state=CommitStatus.pending,
# pagure requires "valid url"
url="",
)
self.create_srpm_if_needed()
if not self.srpm_model.success:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=CommitStatus.failure,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
)
return HandlerResults(success=False, details={"msg": msg})
try:
build_id, web_url = self.run_build()
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all(
state=CommitStatus.error,
description=f"Submit of the build failed: {ex}",
)
return HandlerResults(success=False, details={"error": str(ex)})
for chroot in self.build_targets:
copr_build = CoprBuildModel.get_or_create(
build_id=str(build_id),
commit_sha=self.event.commit_sha,
project_name=self.job_project,
owner=self.job_owner,
web_url=web_url,
target=chroot,
status="pending",
srpm_build=self.srpm_model,
trigger_model=self.event.db_trigger,
)
url = get_copr_build_log_url_from_flask(id_=copr_build.id)
self.report_status_to_all_for_chroot(
state=CommitStatus.pending,
description="Starting RPM build...",
url=url,
chroot=chroot,
)
# release the hounds!
celery_app.send_task(
"task.babysit_copr_build",
args=(build_id,),
countdown=120, # do the first check in 120s
)
return HandlerResults(success=True, details={})
class CoprBuildJobHelper(BaseBuildJobHelper):
job_type_build = JobType.copr_build
job_type_test = JobType.tests
status_name_build: str = "rpm-build"
status_name_test: str = "testing-farm"
def __init__(
self,
service_config: ServiceConfig,
package_config: PackageConfig,
project: GitProject,
metadata: EventData,
db_trigger: AbstractTriggerDbType,
job_config: JobConfig,
targets_override: Optional[Set[str]] = None,
pushgateway: Optional[Pushgateway] = None,
):
super().__init__(
service_config=service_config,
package_config=package_config,
project=project,
metadata=metadata,
db_trigger=db_trigger,
job_config=job_config,
targets_override=targets_override,
pushgateway=pushgateway,
)
self.msg_retrigger: str = MSG_RETRIGGER.format(
job="build",
command="copr-build" if self.job_build else "build",
place="pull request",
)
@property
def default_project_name(self) -> str:
"""
Project name for copr.
* use hostname prefix for non-github service
* replace slash in namespace with dash
* add `-stg` suffix for the stg app
"""
service_hostname = parse_git_repo(
self.project.service.instance_url).hostname
service_prefix = ("" if isinstance(self.project, GithubProject) else
f"{service_hostname}-")
namespace = self.project.namespace.replace("/", "-")
stg = "-stg" if self.service_config.deployment == Deployment.stg else ""
# We want to share project between all releases.
# More details: https://github.com/packit/packit-service/issues/1044
identifier = "releases" if self.metadata.tag_name else self.metadata.identifier
return f"{service_prefix}{namespace}-{self.project.repo}-{identifier}{stg}"
@property
def job_project(self) -> Optional[str]:
"""
The job definition from the config file.
"""
if self.job_build and self.job_build.metadata.project:
return self.job_build.metadata.project
if self.job_tests and self.job_tests.metadata.project:
return self.job_tests.metadata.project
return self.default_project_name
@property
def job_owner(self) -> Optional[str]:
"""
Owner used for the copr build -- search the config or use the copr's config.
"""
if self.job_build and self.job_build.metadata.owner:
return self.job_build.metadata.owner
if self.job_tests and self.job_tests.metadata.owner:
return self.job_tests.metadata.owner
return self.api.copr_helper.copr_client.config.get("username")
@property
def preserve_project(self) -> Optional[bool]:
"""
If the project will be preserved or can be removed after 60 days.
"""
return self.job_build.metadata.preserve_project if self.job_build else None
@property
def list_on_homepage(self) -> Optional[bool]:
"""
If the project will be shown on the copr home page.
"""
return self.job_build.metadata.list_on_homepage if self.job_build else None
@property
def additional_repos(self) -> Optional[List[str]]:
"""
Additional repos that will be enable for copr build.
"""
return self.job_build.metadata.additional_repos if self.job_build else None
@property
def build_targets_all(self) -> Set[str]:
"""
Return all valid Copr build targets/chroots from config.
"""
return get_valid_build_targets(*self.configured_build_targets,
default=None)
@property
def tests_targets_all(self) -> Set[str]:
"""
Return all valid test targets/chroots from config.
"""
return get_valid_build_targets(*self.configured_tests_targets,
default=None)
@property
def available_chroots(self) -> Set[str]:
"""
Returns set of available COPR targets.
"""
return {
*filter(
lambda chroot: not chroot.startswith("_"),
self.api.copr_helper.get_copr_client().mock_chroot_proxy.
get_list().keys(),
)
}
def get_built_packages(self, build_id: int, chroot: str) -> List:
return self.api.copr_helper.copr_client.build_chroot_proxy.get_built_packages(
build_id, chroot).packages
def get_build(self, build_id: int):
return self.api.copr_helper.copr_client.build_proxy.get(build_id)
def monitor_not_submitted_copr_builds(self, number_of_builds: int,
reason: str):
"""
Measure the time it took to set the failed status in case of event (e.g. failed SRPM)
that prevents Copr build to be submitted.
"""
time = measure_time(end=datetime.now(timezone.utc),
begin=self.metadata.task_accepted_time)
for _ in range(number_of_builds):
self.pushgateway.copr_build_not_submitted_time.labels(
reason=reason).observe(time)
def run_copr_build(self) -> TaskResults:
self.report_status_to_all(
description="Building SRPM ...",
state=BaseCommitStatus.running,
# pagure requires "valid url"
url="",
)
if results := self.create_srpm_if_needed():
return results
if not self.srpm_model.success:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=BaseCommitStatus.failure,
description=msg,
url=get_srpm_build_info_url(self.srpm_model.id),
)
self.monitor_not_submitted_copr_builds(len(self.build_targets),
"srpm_failure")
return TaskResults(success=False, details={"msg": msg})
try:
build_id, web_url = self.run_build()
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all(
state=BaseCommitStatus.error,
description=f"Submit of the build failed: {ex}",
)
self.monitor_not_submitted_copr_builds(len(self.build_targets),
"submit_failure")
return TaskResults(
success=False,
details={
"msg": "Submit of the Copr build failed.",
"error": str(ex)
},
)
unprocessed_chroots = []
for chroot in self.build_targets:
if chroot not in self.available_chroots:
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.error,
description=f"Not supported target: {chroot}",
url=get_srpm_build_info_url(self.srpm_model.id),
chroot=chroot,
)
self.monitor_not_submitted_copr_builds(1,
"not_supported_target")
unprocessed_chroots.append(chroot)
continue
copr_build = CoprBuildModel.create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
project_name=self.job_project,
owner=self.job_owner,
web_url=web_url,
target=chroot,
status="pending",
run_model=self.run_model,
task_accepted_time=self.metadata.task_accepted_time,
)
url = get_copr_build_info_url(id_=copr_build.id)
self.report_status_to_all_for_chroot(
state=BaseCommitStatus.running,
description="Starting RPM build...",
url=url,
chroot=chroot,
)
if unprocessed_chroots:
unprocessed = "\n".join(sorted(unprocessed_chroots))
available = "\n".join(sorted(self.available_chroots))
self.status_reporter.comment(
body="There are build targets that are not supported by COPR.\n"
"<details>\n<summary>Unprocessed build targets</summary>\n\n"
f"```\n{unprocessed}\n```\n</details>\n"
"<details>\n<summary>Available build targets</summary>\n\n"
f"```\n{available}\n```\n</details>", )
# release the hounds!
celery_app.send_task(
"task.babysit_copr_build",
args=(build_id, ),
countdown=120, # do the first check in 120s
)
return TaskResults(success=True, details={})
def run_copr_build(self) -> TaskResults:
if not (self.job_build or self.job_tests):
msg = "No copr_build or tests job defined."
# we can't report it to end-user at this stage
return TaskResults(success=False, details={"msg": msg})
self.report_status_to_all(
description="Building SRPM ...",
state=CommitStatus.pending,
# pagure requires "valid url"
url="",
)
self.create_srpm_if_needed()
if not self.srpm_model.success:
msg = "SRPM build failed, check the logs for details."
self.report_status_to_all(
state=CommitStatus.failure,
description=msg,
url=get_srpm_log_url_from_flask(self.srpm_model.id),
)
return TaskResults(success=False, details={"msg": msg})
try:
build_id, web_url = self.run_build()
Pushgateway().push_copr_build_created()
except Exception as ex:
sentry_integration.send_to_sentry(ex)
# TODO: Where can we show more info about failure?
# TODO: Retry
self.report_status_to_all(
state=CommitStatus.error,
description=f"Submit of the build failed: {ex}",
)
return TaskResults(
success=False,
details={"msg": "Submit of the Copr build failed.", "error": str(ex)},
)
unprocessed_chroots = []
for chroot in self.build_targets:
if chroot not in self.available_chroots:
self.report_status_to_all_for_chroot(
state=CommitStatus.error,
description=f"Not supported target: {chroot}",
url=get_srpm_log_url_from_flask(self.srpm_model.id),
chroot=chroot,
)
unprocessed_chroots.append(chroot)
continue
copr_build = CoprBuildModel.get_or_create(
build_id=str(build_id),
commit_sha=self.metadata.commit_sha,
project_name=self.job_project,
owner=self.job_owner,
web_url=web_url,
target=chroot,
status="pending",
srpm_build=self.srpm_model,
trigger_model=self.db_trigger,
)
url = get_copr_build_info_url_from_flask(id_=copr_build.id)
self.report_status_to_all_for_chroot(
state=CommitStatus.pending,
description="Starting RPM build...",
url=url,
chroot=chroot,
)
if unprocessed_chroots:
unprocessed = "\n".join(sorted(unprocessed_chroots))
available = "\n".join(sorted(self.available_chroots))
self.project.pr_comment(
pr_id=self.metadata.pr_id,
body="There are build targets that are not supported by COPR.\n"
"<details>\n<summary>Unprocessed build targets</summary>\n\n"
f"```\n{unprocessed}\n```\n</details>\n"
"<details>\n<summary>Available build targets</summary>\n\n"
f"```\n{available}\n```\n</details>",
)
# release the hounds!
celery_app.send_task(
"task.babysit_copr_build",
args=(build_id,),
countdown=120, # do the first check in 120s
)
return TaskResults(success=True, details={})
