def testSettingsOverrideEnv(self):
settings = get_default_settings()
# With no environment variables, default settings shouldn't be updated
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertDictEqual(settings, updated_settings)
# Invalid frequency
os.environ["RELEASE_FREQUENCY"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_environment(updated_settings)
os.environ["RELEASE_FREQUENCY"] = "nightly"
os.environ["RAY_TEST_REPO"] = "https://github.com/user/ray.git"
os.environ["RAY_TEST_BRANCH"] = "sub/branch"
os.environ["RAY_WHEELS"] = "custom-wheels"
os.environ["TEST_NAME"] = "name_filter"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.NIGHTLY,
"test_name_filter": "name_filter",
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
},
)
def testSettingsOverrideBuildkite(self):
settings = get_default_settings()
with patch(
"ray_release.buildkite.settings.get_buildkite_prompt_value",
self.buildkite_mock,
):
# With no buildkite variables, default settings shouldn't be updated
updated_settings = settings.copy()
update_settings_from_buildkite(updated_settings)
self.assertDictEqual(settings, updated_settings)
# Invalid frequency
self.buildkite["release-frequency"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_buildkite(updated_settings)
# Invalid priority
self.buildkite["release-priority"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_buildkite(updated_settings)
self.buildkite["release-frequency"] = "nightly"
self.buildkite["release-ray-test-repo-branch"] = "user:sub/branch"
self.buildkite["release-ray-wheels"] = "custom-wheels"
self.buildkite["release-test-name"] = "name_filter"
self.buildkite["release-priority"] = "manual"
updated_settings = settings.copy()
update_settings_from_buildkite(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.NIGHTLY,
"test_name_filter": "name_filter",
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
"priority": Priority.MANUAL,
"no_concurrency_limit": False,
},
)
def testSettingsOverrideBuildkite(self):
settings = get_default_settings()
with patch(
"ray_release.buildkite.settings.get_buildkite_prompt_value",
self.buildkite_mock,
):
# With no buildkite variables, default settings shouldn't be updated
updated_settings = settings.copy()
update_settings_from_buildkite(updated_settings)
self.assertDictEqual(settings, updated_settings)
buildkite = self.buildkite.copy()
# Invalid frequency
self.buildkite.clear()
self.buildkite.update(buildkite)
self.buildkite["release-frequency"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_buildkite(updated_settings)
# Invalid priority
self.buildkite.clear()
self.buildkite.update(buildkite)
self.buildkite["release-priority"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_buildkite(updated_settings)
# Invalid test attr regex filters
self.buildkite.clear()
self.buildkite.update(buildkite)
self.buildkite["release-test-attr-regex-filters"] = "xxxx"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_buildkite(updated_settings)
self.buildkite.clear()
self.buildkite.update(buildkite)
self.buildkite[
"release-test-attr-regex-filters"] = "name:xxx\ngroup:yyy"
updated_settings = settings.copy()
update_settings_from_buildkite(updated_settings)
self.assertDictEqual(
updated_settings["test_attr_regex_filters"],
{
"name": "xxx",
"group": "yyy",
},
)
self.buildkite.clear()
self.buildkite.update(buildkite)
self.buildkite["release-frequency"] = "nightly"
self.buildkite["release-ray-test-repo-branch"] = "user:sub/branch"
self.buildkite["release-ray-wheels"] = "custom-wheels"
self.buildkite["release-test-name"] = "name_filter"
self.buildkite["release-priority"] = "manual"
updated_settings = settings.copy()
update_settings_from_buildkite(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.NIGHTLY,
"prefer_smoke_tests": False,
"test_attr_regex_filters": {
"name": "name_filter"
},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
"priority": Priority.MANUAL,
"no_concurrency_limit": False,
},
)
self.buildkite["release-frequency"] = "any-smoke"
update_settings_from_buildkite(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.ANY,
"prefer_smoke_tests": True,
"test_attr_regex_filters": {
"name": "name_filter"
},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
"priority": Priority.MANUAL,
"no_concurrency_limit": False,
},
)
def testSettingsOverrideEnv(self):
settings = get_default_settings()
# With no environment variables, default settings shouldn't be updated
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertDictEqual(settings, updated_settings)
environ = os.environ.copy()
# Invalid frequency
os.environ.clear()
os.environ.update(environ)
os.environ["RELEASE_FREQUENCY"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_environment(updated_settings)
# Invalid priority
os.environ.clear()
os.environ.update(environ)
os.environ["RELEASE_PRIORITY"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_environment(updated_settings)
# Invalid test attr regex filters
os.environ.clear()
os.environ.update(environ)
os.environ["TEST_ATTR_REGEX_FILTERS"] = "xxxx"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_environment(updated_settings)
os.environ.clear()
os.environ.update(environ)
os.environ["TEST_ATTR_REGEX_FILTERS"] = "name:xxx\nteam:yyy\n"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertDictEqual(
updated_settings["test_attr_regex_filters"],
{
"name": "xxx",
"team": "yyy",
},
)
os.environ.clear()
os.environ.update(environ)
os.environ["RELEASE_FREQUENCY"] = "nightly"
os.environ["RAY_TEST_REPO"] = "https://github.com/user/ray.git"
os.environ["RAY_TEST_BRANCH"] = "sub/branch"
os.environ["RAY_WHEELS"] = "custom-wheels"
os.environ["TEST_NAME"] = "name_filter"
os.environ["RELEASE_PRIORITY"] = "manual"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.NIGHTLY,
"prefer_smoke_tests": False,
"test_attr_regex_filters": {
"name": "name_filter"
},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
"priority": Priority.MANUAL,
"no_concurrency_limit": False,
},
)
os.environ["RELEASE_FREQUENCY"] = "any-smoke"
update_settings_from_environment(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.ANY,
"prefer_smoke_tests": True,
"test_attr_regex_filters": {
"name": "name_filter"
},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
"priority": Priority.MANUAL,
"no_concurrency_limit": False,
},
)
###
# Buildkite PR build settings
# Default PR
os.environ.clear()
os.environ.update(environ)
os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
os.environ[
"BUILDKITE_PULL_REQUEST_REPO"] = "https://github.com/user/ray-fork.git"
os.environ["BUILDKITE_BRANCH"] = "user:some_branch"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertEqual(updated_settings["ray_test_repo"],
"https://github.com/user/ray-fork.git")
self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
# PR without prefix
os.environ.clear()
os.environ.update(environ)
os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
os.environ[
"BUILDKITE_PULL_REQUEST_REPO"] = "git://github.com/user/ray-fork.git"
os.environ["BUILDKITE_BRANCH"] = "some_branch"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertEqual(updated_settings["ray_test_repo"],
"https://github.com/user/ray-fork.git")
self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
# Branch build but pointing to fork
os.environ.clear()
os.environ.update(environ)
os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
os.environ["BUILDKITE_BRANCH"] = "user:some_branch"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertEqual(updated_settings["ray_test_repo"],
"https://github.com/user/ray.git")
self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
# Branch build but pointing to main repo branch
os.environ.clear()
os.environ.update(environ)
os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
os.environ["BUILDKITE_BRANCH"] = "some_branch"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertEqual(updated_settings["ray_test_repo"],
"https://github.com/ray-project/ray.git")
self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
# Empty BUILDKITE_PULL_REQUEST_REPO
os.environ.clear()
os.environ.update(environ)
os.environ["BUILDKITE_REPO"] = "https://github.com/ray-project/ray.git"
os.environ["BUILDKITE_BRANCH"] = "some_branch"
os.environ["BUILDKITE_PULL_REQUEST_REPO"] = ""
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertEqual(updated_settings["ray_test_repo"],
"https://github.com/ray-project/ray.git")
self.assertEqual(updated_settings["ray_test_branch"], "some_branch")
def testSettingsOverrideEnv(self):
settings = get_default_settings()
# With no environment variables, default settings shouldn't be updated
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertDictEqual(settings, updated_settings)
environ = os.environ.copy()
# Invalid frequency
os.environ.clear()
os.environ.update(environ)
os.environ["RELEASE_FREQUENCY"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_environment(updated_settings)
# Invalid priority
os.environ.clear()
os.environ.update(environ)
os.environ["RELEASE_PRIORITY"] = "invalid"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_environment(updated_settings)
# Invalid test attr regex filters
os.environ.clear()
os.environ.update(environ)
os.environ["TEST_ATTR_REGEX_FILTERS"] = "xxxx"
updated_settings = settings.copy()
with self.assertRaises(ReleaseTestConfigError):
update_settings_from_environment(updated_settings)
os.environ.clear()
os.environ.update(environ)
os.environ["TEST_ATTR_REGEX_FILTERS"] = "name:xxx\nteam:yyy\n"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertDictEqual(
updated_settings["test_attr_regex_filters"],
{
"name": "xxx",
"team": "yyy",
},
)
os.environ.clear()
os.environ.update(environ)
os.environ["RELEASE_FREQUENCY"] = "nightly"
os.environ["RAY_TEST_REPO"] = "https://github.com/user/ray.git"
os.environ["RAY_TEST_BRANCH"] = "sub/branch"
os.environ["RAY_WHEELS"] = "custom-wheels"
os.environ["TEST_NAME"] = "name_filter"
os.environ["RELEASE_PRIORITY"] = "manual"
updated_settings = settings.copy()
update_settings_from_environment(updated_settings)
self.assertDictEqual(
updated_settings,
{
"frequency": Frequency.NIGHTLY,
"test_attr_regex_filters": {
"name": "name_filter"
},
"ray_wheels": "custom-wheels",
"ray_test_repo": "https://github.com/user/ray.git",
"ray_test_branch": "sub/branch",
"priority": Priority.MANUAL,
"no_concurrency_limit": False,
},
)