def handle(self, *args: Any, **options: Any) -> None:
errors = 0
publisher = get_publisher()
# Pass 1: check build content
records = publisher.records.query(completed__isnull=False)
for record in records:
missing: list[Path] = []
for content in Content:
path = publisher.storage.get_path(record, content)
if not path.exists():
missing.append(path)
if missing:
self.stderr.write(f"Path missing for {record}: {missing}")
errors += 1
# Pass 2: check for orphans
for content in Content:
directory = publisher.storage.path / content.value
for path in directory.glob("*.*"):
build = Build(path.name)
try:
publisher.records.get(build)
except RecordNotFound:
self.stderr.write(f"Record missing for {path}")
errors += 1
if errors:
self.stderr.write("Errors were encountered.")
raise CommandError(errors)
def resolve_mutation_pull(_obj: Any, _info: GraphQLResolveInfo,
id: str) -> MachineInfo:
build = Build(id)
pull_build.delay(id)
return MachineInfo(build.machine)
def pull_build(build_id: str) -> None:
"""Pull the build into storage"""
publisher = get_publisher()
build = Build(build_id)
try:
publisher.pull(build)
except Exception as error:
logger.exception("Failed to pull build %s", build)
# If this is an error due to 404 response don't retry
if isinstance(error, requests.exceptions.HTTPError):
response = getattr(error, "response", None)
if response and response.status_code == 404:
publisher.records.delete(build)
raise
if isinstance(error, PULL_RETRYABLE_EXCEPTIONS):
pull_build.retry(exc=error)
return
publisher.records.delete(build)
raise
if Settings.from_environ().ENABLE_PURGE:
purge_build.delay(build.machine)
def test_pulls_build(self):
"""Should actually pull the build"""
with mock.patch("gentoo_build_publisher.tasks.purge_build"):
pull_build.s("lima.1012").apply()
build = Build("lima.1012")
self.assertIs(self.publisher.pulled(build), True)
def test_publishes_build(self):
"""Should actually publish the build"""
with mock.patch("gentoo_build_publisher.tasks.purge_build"):
result = publish_build.s("babette.193").apply()
build = Build("babette.193")
self.assertIs(self.publisher.published(build), True)
self.assertIs(result.result, True)
def delete_build(build_id: str) -> None:
"""Delete the given build from the db"""
publisher = get_publisher()
build = Build(build_id)
logger.info("Deleting build: %s", build)
publisher.delete(build)
logger.info("Deleted build: %s", build)
def published_build(self) -> Build | None:
publisher = get_publisher()
try:
return next(
Build(build.id) for build in self.builds
if publisher.published(build))
except StopIteration:
return None
def resolve_mutation_publish(_obj: Any, _info: GraphQLResolveInfo,
id: str) -> MachineInfo:
publisher = get_publisher()
build = Build(id)
if publisher.pulled(build):
publisher.publish(build)
else:
publish_build.delay(build.id)
return MachineInfo(build.machine)
def resolve_query_diff(_obj: Any, _info: GraphQLResolveInfo, left: str,
right: str) -> Optional[Object]:
publisher = get_publisher()
left_build = Build(left)
if not publisher.records.exists(left_build):
return None
right_build = Build(right)
if not publisher.records.exists(right_build):
return None
items = publisher.diff_binpkgs(left_build, right_build)
return {
"left": BuildType(left_build),
"right": BuildType(right_build),
"items": [*items],
}
def setUp(self):
super().setUp()
self.records = RecordDB()
self.build_model = BuildModelFactory.create(
submitted=dt.datetime(2022, 2, 20, 15, 47, tzinfo=dt.timezone.utc),
completed=dt.datetime(2022, 2, 20, 15, 58, tzinfo=dt.timezone.utc),
built=dt.datetime(2022, 2, 20, 15, 58, tzinfo=dt.timezone.utc),
)
BuildLog.objects.create(build_model=self.build_model, logs="This is a test")
self.record = self.records.get(Build(str(self.build_model)))
def test_should_delete_db_model_when_download_fails(self):
settings = Settings.from_environ()
records = Records.from_settings(settings)
with mock.patch(
"gentoo_build_publisher.publisher.Jenkins.download_artifact"
) as download_artifact_mock:
download_artifact_mock.side_effect = Exception
pull_build.s("oscar.197").apply()
with self.assertRaises(RecordNotFound):
records.get(Build("oscar.197"))
def resolve_mutation_releasebuild(_obj: Any, _info: GraphQLResolveInfo,
id: str) -> Optional[BuildType]:
publisher = get_publisher()
build = Build(id)
if not publisher.records.exists(build):
return None
record = publisher.record(build)
publisher.records.save(record, keep=False)
return BuildType(record)
def publish_build(build_id: str) -> bool:
"""Publish the build"""
publisher = get_publisher()
try:
pull_build.apply((build_id, ), throw=True)
except PUBLISH_FATAL_EXCEPTIONS:
logger.error("Build %s failed to pull. Not publishing", f"{build_id}")
return False
publisher.publish(Build(build_id))
return True
def resolve_mutation_createnote(
_obj: Any,
_info: GraphQLResolveInfo,
id: str,
note: Optional[str] = None) -> Optional[BuildType]:
publisher = get_publisher()
build = Build(id)
if not publisher.records.exists(build):
return None
record = publisher.record(build)
publisher.records.save(record, note=note)
return BuildType(record)
def test_download_artifact_with_no_auth(self):
# Given the build id
build = Build("babette.193")
# Given the Jenkins instance having no user/api_key
jenkins = MockJenkins(JENKINS_CONFIG)
# When we call download_artifact on the build
jenkins.download_artifact(build)
# Then it requests the artifact with no auth
jenkins.mock_get.assert_called_with(
"https://jenkins.invalid/job/babette/193/artifact/build.tar.gz",
auth=jenkins.config.auth(),
stream=True,
)
def test_get_metadata(self, mock_requests_get):
mock_response = test_data("jenkins_build.json")
mock_requests_get.return_value.json.return_value = json.loads(
mock_response)
build = Build("babette.291")
jenkins = Jenkins(JENKINS_CONFIG)
metadata = jenkins.get_metadata(build)
self.assertEqual(
metadata, JenkinsMetadata(duration=3892427,
timestamp=1635811517838))
mock_requests_get.assert_called_once_with(
"https://jenkins.invalid/job/babette/291/api/json",
auth=("jenkins", "foo"))
mock_requests_get.return_value.json.assert_called_once_with()
def test_artifact_url(self):
""".build_url() should return the url of the given build artifact"""
# Given the build id
build = Build("babette.193")
# Given the Jenkins instance
jenkins = Jenkins(JENKINS_CONFIG)
# When we call .build_url
build_url = jenkins.artifact_url(build)
# Then we get the expected url
self.assertEqual(
build_url,
URL("https://jenkins.invalid/job/babette/193/artifact/build.tar.gz"
),
)
def test_download_artifact(self):
""".download_artifact should download the given build artifact"""
# Given the build id
build = Build("babette.193")
# Given the Jenkins instance
jenkins = MockJenkins(JENKINS_CONFIG)
# When we call download_artifact on the build
stream = jenkins.download_artifact(build)
# Then it streams the build artifact's contents
bytes_io = io.BytesIO()
for chunk in stream:
bytes_io.write(chunk)
jenkins.mock_get.assert_called_with(
"https://jenkins.invalid/job/babette/193/artifact/build.tar.gz",
auth=("jenkins", "foo"),
stream=True,
)
def test_repr(self):
build = Build("babette.16")
self.assertEqual("Build('babette.16')", repr(build))
def test_get_does_not_exist(self):
with self.assertRaises(RecordNotFound):
self.records.get(Build("bogus.955"))
def test_get(self):
build = Build(str(self.build_model))
record = self.records.get(build)
self.assertEqual(record.id, build.id)
self.assertEqual(record.submitted, self.build_model.submitted)
def test_should_delete_the_build(self):
with mock.patch.object(self.publisher, "delete") as mock_delete:
delete_build.s("zulu.56").apply()
mock_delete.assert_called_once_with(Build("zulu.56"))
def test_string_with_name_and_number(self):
build = Build("babette.16")
self.assertEqual(str(build), "babette.16")
def resolve_query_build(_obj: Any, _info: GraphQLResolveInfo,
id: str) -> Optional[BuildType]:
publisher = get_publisher()
build = Build(id)
return None if not publisher.records.exists(build) else BuildType(build)
def test_has_machine_and_build_id_attrs(self):
build = Build("babette.16")
self.assertEqual(build.machine, "babette")
self.assertEqual(build.build_id, "16")
def test_string_with_no_name(self):
with self.assertRaises(InvalidBuild):
Build(".16")
