def handle(self, *args: Any, **options: Any) -> None: errors = 0 publisher = get_publisher() # Pass 1: check build content records = publisher.records.query(completed__isnull=False) for record in records: missing: list[Path] = [] for content in Content: path = publisher.storage.get_path(record, content) if not path.exists(): missing.append(path) if missing: self.stderr.write(f"Path missing for {record}: {missing}") errors += 1 # Pass 2: check for orphans for content in Content: directory = publisher.storage.path / content.value for path in directory.glob("*.*"): build = Build(path.name) try: publisher.records.get(build) except RecordNotFound: self.stderr.write(f"Record missing for {path}") errors += 1 if errors: self.stderr.write("Errors were encountered.") raise CommandError(errors)
def resolve_mutation_pull(_obj: Any, _info: GraphQLResolveInfo, id: str) -> MachineInfo: build = Build(id) pull_build.delay(id) return MachineInfo(build.machine)
def pull_build(build_id: str) -> None: """Pull the build into storage""" publisher = get_publisher() build = Build(build_id) try: publisher.pull(build) except Exception as error: logger.exception("Failed to pull build %s", build) # If this is an error due to 404 response don't retry if isinstance(error, requests.exceptions.HTTPError): response = getattr(error, "response", None) if response and response.status_code == 404: publisher.records.delete(build) raise if isinstance(error, PULL_RETRYABLE_EXCEPTIONS): pull_build.retry(exc=error) return publisher.records.delete(build) raise if Settings.from_environ().ENABLE_PURGE: purge_build.delay(build.machine)
def test_pulls_build(self): """Should actually pull the build""" with mock.patch("gentoo_build_publisher.tasks.purge_build"): pull_build.s("lima.1012").apply() build = Build("lima.1012") self.assertIs(self.publisher.pulled(build), True)
def test_publishes_build(self): """Should actually publish the build""" with mock.patch("gentoo_build_publisher.tasks.purge_build"): result = publish_build.s("babette.193").apply() build = Build("babette.193") self.assertIs(self.publisher.published(build), True) self.assertIs(result.result, True)
def delete_build(build_id: str) -> None: """Delete the given build from the db""" publisher = get_publisher() build = Build(build_id) logger.info("Deleting build: %s", build) publisher.delete(build) logger.info("Deleted build: %s", build)
def published_build(self) -> Build | None: publisher = get_publisher() try: return next( Build(build.id) for build in self.builds if publisher.published(build)) except StopIteration: return None
def resolve_mutation_publish(_obj: Any, _info: GraphQLResolveInfo, id: str) -> MachineInfo: publisher = get_publisher() build = Build(id) if publisher.pulled(build): publisher.publish(build) else: publish_build.delay(build.id) return MachineInfo(build.machine)
def resolve_query_diff(_obj: Any, _info: GraphQLResolveInfo, left: str, right: str) -> Optional[Object]: publisher = get_publisher() left_build = Build(left) if not publisher.records.exists(left_build): return None right_build = Build(right) if not publisher.records.exists(right_build): return None items = publisher.diff_binpkgs(left_build, right_build) return { "left": BuildType(left_build), "right": BuildType(right_build), "items": [*items], }
def setUp(self): super().setUp() self.records = RecordDB() self.build_model = BuildModelFactory.create( submitted=dt.datetime(2022, 2, 20, 15, 47, tzinfo=dt.timezone.utc), completed=dt.datetime(2022, 2, 20, 15, 58, tzinfo=dt.timezone.utc), built=dt.datetime(2022, 2, 20, 15, 58, tzinfo=dt.timezone.utc), ) BuildLog.objects.create(build_model=self.build_model, logs="This is a test") self.record = self.records.get(Build(str(self.build_model)))
def test_should_delete_db_model_when_download_fails(self): settings = Settings.from_environ() records = Records.from_settings(settings) with mock.patch( "gentoo_build_publisher.publisher.Jenkins.download_artifact" ) as download_artifact_mock: download_artifact_mock.side_effect = Exception pull_build.s("oscar.197").apply() with self.assertRaises(RecordNotFound): records.get(Build("oscar.197"))
def resolve_mutation_releasebuild(_obj: Any, _info: GraphQLResolveInfo, id: str) -> Optional[BuildType]: publisher = get_publisher() build = Build(id) if not publisher.records.exists(build): return None record = publisher.record(build) publisher.records.save(record, keep=False) return BuildType(record)
def publish_build(build_id: str) -> bool: """Publish the build""" publisher = get_publisher() try: pull_build.apply((build_id, ), throw=True) except PUBLISH_FATAL_EXCEPTIONS: logger.error("Build %s failed to pull. Not publishing", f"{build_id}") return False publisher.publish(Build(build_id)) return True
def resolve_mutation_createnote( _obj: Any, _info: GraphQLResolveInfo, id: str, note: Optional[str] = None) -> Optional[BuildType]: publisher = get_publisher() build = Build(id) if not publisher.records.exists(build): return None record = publisher.record(build) publisher.records.save(record, note=note) return BuildType(record)
def test_download_artifact_with_no_auth(self): # Given the build id build = Build("babette.193") # Given the Jenkins instance having no user/api_key jenkins = MockJenkins(JENKINS_CONFIG) # When we call download_artifact on the build jenkins.download_artifact(build) # Then it requests the artifact with no auth jenkins.mock_get.assert_called_with( "https://jenkins.invalid/job/babette/193/artifact/build.tar.gz", auth=jenkins.config.auth(), stream=True, )
def test_get_metadata(self, mock_requests_get): mock_response = test_data("jenkins_build.json") mock_requests_get.return_value.json.return_value = json.loads( mock_response) build = Build("babette.291") jenkins = Jenkins(JENKINS_CONFIG) metadata = jenkins.get_metadata(build) self.assertEqual( metadata, JenkinsMetadata(duration=3892427, timestamp=1635811517838)) mock_requests_get.assert_called_once_with( "https://jenkins.invalid/job/babette/291/api/json", auth=("jenkins", "foo")) mock_requests_get.return_value.json.assert_called_once_with()
def test_artifact_url(self): """.build_url() should return the url of the given build artifact""" # Given the build id build = Build("babette.193") # Given the Jenkins instance jenkins = Jenkins(JENKINS_CONFIG) # When we call .build_url build_url = jenkins.artifact_url(build) # Then we get the expected url self.assertEqual( build_url, URL("https://jenkins.invalid/job/babette/193/artifact/build.tar.gz" ), )
def test_download_artifact(self): """.download_artifact should download the given build artifact""" # Given the build id build = Build("babette.193") # Given the Jenkins instance jenkins = MockJenkins(JENKINS_CONFIG) # When we call download_artifact on the build stream = jenkins.download_artifact(build) # Then it streams the build artifact's contents bytes_io = io.BytesIO() for chunk in stream: bytes_io.write(chunk) jenkins.mock_get.assert_called_with( "https://jenkins.invalid/job/babette/193/artifact/build.tar.gz", auth=("jenkins", "foo"), stream=True, )
def test_repr(self): build = Build("babette.16") self.assertEqual("Build('babette.16')", repr(build))
def test_get_does_not_exist(self): with self.assertRaises(RecordNotFound): self.records.get(Build("bogus.955"))
def test_get(self): build = Build(str(self.build_model)) record = self.records.get(build) self.assertEqual(record.id, build.id) self.assertEqual(record.submitted, self.build_model.submitted)
def test_should_delete_the_build(self): with mock.patch.object(self.publisher, "delete") as mock_delete: delete_build.s("zulu.56").apply() mock_delete.assert_called_once_with(Build("zulu.56"))
def test_string_with_name_and_number(self): build = Build("babette.16") self.assertEqual(str(build), "babette.16")
def resolve_query_build(_obj: Any, _info: GraphQLResolveInfo, id: str) -> Optional[BuildType]: publisher = get_publisher() build = Build(id) return None if not publisher.records.exists(build) else BuildType(build)
def test_has_machine_and_build_id_attrs(self): build = Build("babette.16") self.assertEqual(build.machine, "babette") self.assertEqual(build.build_id, "16")
def test_string_with_no_name(self): with self.assertRaises(InvalidBuild): Build(".16")