def test_get_broker(self):
os.environ["POLYAXON_ENVIRONMENT"] = "testing"
os.environ.pop("POLYAXON_BROKER_BACKEND", None)
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json"
])
assert config.broker_backend == "rabbitmq"
assert config.is_redis_broker is False
assert config.is_rabbitmq_broker is True
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_BROKER_BACKEND": "rabbitmq"
},
])
assert config.broker_backend == "rabbitmq"
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_BROKER_BACKEND": "redis"
},
])
assert config.broker_backend == "redis"
assert config.is_redis_broker is True
assert config.is_rabbitmq_broker is False
def set_cors(context, config: ConfigManager):
# session settings
context["CORS_ALLOW_CREDENTIALS"] = True
whitelist = config.get_list("POLYAXON_CORS_ORIGIN_WHITELIST",
is_optional=True,
default=[])
context["CORS_ORIGIN_WHITELIST"] = whitelist
context["CORS_ORIGIN_ALLOW_ALL"] = False if whitelist else True
context["CORS_ALLOW_HEADERS"] = default_headers + (
PolyaxonServiceHeaders.CLI_VERSION,
PolyaxonServiceHeaders.CLIENT_VERSION,
PolyaxonServiceHeaders.INTERNAL,
PolyaxonServiceHeaders.SERVICE,
)
ssl_enabled = config.get_boolean("POLYAXON_SSL_ENABLED",
is_optional=True,
default=False)
ssl_redirect_enabled = config.get_boolean("POLYAXON_SSL_REDIRECT_ENABLED",
is_optional=True,
default=False)
context["SSL_ENABLED"] = ssl_enabled
context["PROTOCOL"] = "http"
context["WS_PROTOCOL"] = "ws"
if ssl_enabled:
context["SESSION_COOKIE_SECURE"] = True
context["CSRF_COOKIE_SECURE"] = True
context["SECURE_PROXY_SSL_HEADER"] = ("HTTP_X_FORWARDED_PROTO",
"https")
context["PROTOCOL"] = "https"
context["WS_PROTOCOL"] = "wss"
if ssl_redirect_enabled:
context["SECURE_SSL_REDIRECT"] = True
def set_encryption(context, config: ConfigManager):
context["ENCRYPTION_KEY"] = config.get_string("POLYAXON_ENCRYPTION_KEY",
is_optional=True)
context["ENCRYPTION_SECRET"] = config.get_string(
"POLYAXON_ENCRYPTION_SECRET", is_optional=True)
context["ENCRYPTION_BACKEND"] = config.get_string(
"POLYAXON_ENCRYPTION_BACKEND", is_optional=True)
def set_cors(context, config: ConfigManager):
# session settings
context["CORS_ALLOW_CREDENTIALS"] = True
allowed_list = config.get_list("POLYAXON_CORS_ALLOWED_ORIGINS",
is_optional=True,
default=[])
context["CORS_ALLOWED_ORIGINS"] = allowed_list
context["CORS_ALLOW_ALL_ORIGINS"] = False if allowed_list else True
context["CORS_ALLOW_HEADERS"] = (default_headers +
PolyaxonServiceHeaders.get_headers())
ssl_enabled = config.get_boolean("POLYAXON_SSL_ENABLED",
is_optional=True,
default=False)
ssl_redirect_enabled = config.get_boolean("POLYAXON_SSL_REDIRECT_ENABLED",
is_optional=True,
default=False)
context["SSL_ENABLED"] = ssl_enabled
context["PROTOCOL"] = "http"
context["WS_PROTOCOL"] = "ws"
if ssl_enabled:
context["SESSION_COOKIE_SECURE"] = True
context["CSRF_COOKIE_SECURE"] = True
context["SECURE_PROXY_SSL_HEADER"] = ("HTTP_X_FORWARDED_PROTO",
"https")
context["PROTOCOL"] = "https"
context["WS_PROTOCOL"] = "wss"
if ssl_redirect_enabled:
context["SECURE_SSL_REDIRECT"] = True
def set_api(context, config: ConfigManager, processors: List[str] = None):
context["ROOT_URLCONF"] = "polyconf.urls"
def get_allowed_hosts():
allowed_hosts = config.get_string("POLYAXON_ALLOWED_HOSTS",
is_optional=True,
is_list=True,
default=["*"])
allowed_hosts.append(".polyaxon.com")
platform_host = config.get_string("POLYAXON_PLATFORM_HOST",
is_optional=True)
if platform_host:
allowed_hosts.append(platform_host)
return allowed_hosts
context["ALLOWED_HOSTS"] = get_allowed_hosts()
processors = processors or []
processors = [
"django.contrib.auth.context_processors.auth",
"django.template.context_processors.debug",
"django.template.context_processors.media",
"django.template.context_processors.static",
"django.template.context_processors.tz",
"django.contrib.messages.context_processors.messages",
"polycommon.settings.context_processors.versions",
"polycommon.settings.context_processors.ui_offline",
"polycommon.settings.context_processors.ui_enabled",
] + processors
context["FRONTEND_DEBUG"] = config.get_boolean("POLYAXON_FRONTEND_DEBUG")
template_debug = (config.get_boolean("DJANGO_TEMPLATE_DEBUG",
is_optional=True)
or config.is_debug_mode)
context["UI_ADMIN_ENABLED"] = config.get_boolean(
POLYAXON_KEYS_UI_ADMIN_ENABLED, is_optional=True, default=False)
context["UI_OFFLINE"] = config.get_boolean(POLYAXON_KEYS_UI_OFFLINE,
is_optional=True,
default=False)
context["UI_ENABLED"] = config.get_boolean(POLYAXON_KEYS_UI_ENABLED,
is_optional=True,
default=True)
context["TEMPLATES_DEBUG"] = template_debug
context["LIST_TEMPLATE_CONTEXT_PROCESSORS"] = processors
context["TEMPLATES"] = [{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"APP_DIRS": True,
"OPTIONS": {
"debug": template_debug,
"context_processors": processors
},
}]
def set_apps(
context,
config: ConfigManager,
default_apps: Tuple,
third_party_apps: Optional[Tuple],
project_apps: Tuple,
):
extra_apps = config.get_string(
"POLYAXON_EXTRA_APPS", is_list=True, is_optional=True
)
extra_apps = tuple(extra_apps) if extra_apps else ()
apps = (
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
)
third_party_apps = third_party_apps or ()
context["INSTALLED_APPS"] = (
apps + third_party_apps + default_apps + extra_apps + project_apps
)
def set_admin(context, config: ConfigManager):
admin_name = config.get_string("POLYAXON_ADMIN_NAME", is_optional=True)
admin_mail = config.get_string("POLYAXON_ADMIN_MAIL", is_optional=True)
if admin_mail and admin_mail:
admins = ((admin_name, admin_mail), )
context["ADMINS"] = admins
context["MANAGERS"] = admins
def set_assets(context, root_dir, config: ConfigManager):
context["MEDIA_ROOT"] = config.get_string("POLYAXON_MEDIA_ROOT")
context["MEDIA_URL"] = config.get_string("POLYAXON_MEDIA_URL")
context["STATIC_ROOT"] = config.get_string("POLYAXON_STATIC_ROOT")
context["STATIC_URL"] = config.get_string("POLYAXON_STATIC_URL")
# Additional locations of static files
context["STATICFILES_DIRS"] = (str(root_dir.parent.parent / "public"), )
context["STATICFILES_FINDERS"] = (
"django.contrib.staticfiles.finders.FileSystemFinder",
"django.contrib.staticfiles.finders.AppDirectoriesFinder",
)
context["LOCALE_PATHS"] = (
str(root_dir / "locale"),
str(root_dir / "client" / "js" / "libs" / "locale"),
)
context["STATICI18N_ROOT"] = "static"
context["STATICI18N_OUTPUT_DIR"] = "jsi18n"
context["ARTIFACTS_ROOT"] = config.get_string(
"POLYAXON_ARTIFACTS_ROOT",
is_optional=True,
default="/tmp/plx/artifacts_uploads",
)
context["LOGS_ROOT"] = config.get_string("POLYAXON_LOGS_ROOT",
is_optional=True,
default="/tmp/plx/logs_uploads")
context["ARCHIVES_ROOT"] = config.get_string("POLYAXON_ARCHIVES_ROOT",
is_optional=True,
default="/tmp/plx/archives")
def test_get_from_os_env(self):
os.environ["POLYAXON_ENVIRONMENT"] = "testing"
os.environ["FOO_BAR_KEY"] = "foo_bar"
config = ConfigManager.read_configs(
[os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json"]
)
assert config.get_string("POLYAXON_ENVIRONMENT") == "testing"
assert config.get_string("FOO_BAR_KEY") == "foo_bar"
def set_core(context, config: ConfigManager):
context["DEBUG"] = config.is_debug_mode
context["POLYAXON_SERVICE"] = config.service
context["POLYAXON_ENVIRONMENT"] = config.env
context["CHART_VERSION"] = config.chart_version
context["SCHEDULER_ENABLED"] = config.scheduler_enabled
context["K8S_NAMESPACE"] = config.namespace
context["FILE_UPLOAD_PERMISSIONS"] = 0o644
context["WSGI_APPLICATION"] = "polyconf.wsgi.application"
context["TIME_ZONE"] = config.timezone
context["LANGUAGE_CODE"] = "en"
context["LANGUAGES"] = (("en", u"English"), )
context["USE_I18N"] = True
context["USE_L10N"] = True
context["USE_TZ"] = True
context["INTERNAL_IPS"] = ("127.0.0.1", )
context["APPEND_SLASH"] = True
context["ROOT_URLCONF"] = ""
db_engine = "django.db.backends.postgresql"
context["DEFAULT_DB_ENGINE"] = db_engine
db_definition = {
"ENGINE":
config.get_string("POLYAXON_DB_ENGINE",
is_optional=True,
default=db_engine),
"NAME":
config.get_string("POLYAXON_DB_NAME"),
"USER":
config.get_string("POLYAXON_DB_USER"),
"PASSWORD":
config.get_string("POLYAXON_DB_PASSWORD", is_secret=True),
"HOST":
config.get_string("POLYAXON_DB_HOST"),
"PORT":
config.get_string("POLYAXON_DB_PORT"),
"ATOMIC_REQUESTS":
True,
"CONN_MAX_AGE":
config.get_int("POLYAXON_DB_CONN_MAX_AGE",
is_optional=True,
default=None),
}
db_options = config.get_dict("POLYAXON_DB_OPTIONS",
is_optional=True,
default={})
if db_options:
db_definition["OPTIONS"] = db_options
context["DATABASES"] = {"default": db_definition}
def test_get_broker_url(self):
os.environ["POLYAXON_ENVIRONMENT"] = "testing"
os.environ.pop("POLYAXON_RABBITMQ_USER", None)
os.environ.pop("POLYAXON_RABBITMQ_PASSWORD", None)
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_BROKER_BACKEND": "redis",
"POLYAXON_REDIS_CELERY_BROKER_URL": "foo",
},
])
assert config.get_broker_url() == "redis://foo"
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_REDIS_PROTOCOL": "rediss",
"POLYAXON_BROKER_BACKEND": "redis",
"POLYAXON_REDIS_CELERY_BROKER_URL": "foo",
},
])
assert config.get_broker_url() == "rediss://foo"
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_BROKER_BACKEND": "redis",
"POLYAXON_REDIS_CELERY_BROKER_URL": "foo",
"POLYAXON_REDIS_PASSWORD": "******",
},
])
assert config.get_broker_url() == "redis://:pass@foo"
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_AMQP_URL": "foo",
"POLYAXON_BROKER_BACKEND": "rabbitmq",
"POLYAXON_REDIS_CELERY_BROKER_URL": "foo",
},
])
assert config.get_broker_url() == "amqp://foo"
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_AMQP_URL": "foo",
"POLYAXON_BROKER_BACKEND": "rabbitmq",
"POLYAXON_RABBITMQ_PASSWORD": "",
"POLYAXON_REDIS_CELERY_BROKER_URL": "foo",
},
])
assert config.get_broker_url() == "amqp://foo"
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_AMQP_URL": "foo",
"POLYAXON_BROKER_BACKEND": "rabbitmq",
"POLYAXON_RABBITMQ_PASSWORD": "",
"POLYAXON_RABBITMQ_USER": "******",
},
])
assert config.get_broker_url() == "amqp://foo"
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_AMQP_URL": "foo",
"POLYAXON_BROKER_BACKEND": "rabbitmq",
"POLYAXON_RABBITMQ_USER": "",
"POLYAXON_RABBITMQ_PASSWORD": "******",
},
])
assert config.get_broker_url() == "amqp://foo"
config = ConfigManager.read_configs([
os.environ,
"./tests/fixtures_static/configs/non_opt_config_tests.json",
{
"POLYAXON_AMQP_URL": "foo",
"POLYAXON_BROKER_BACKEND": "rabbitmq",
"POLYAXON_RABBITMQ_USER": "******",
"POLYAXON_RABBITMQ_PASSWORD": "******",
},
])
assert config.get_broker_url() == "amqp://*****:*****@foo"
def test_raises_for_non_optional_env_vars(self):
with self.assertRaises(PolyaxonSchemaError):
ConfigManager.read_configs([os.environ])
def set_celery(context, config: ConfigManager, routes: Dict):
context["CELERY_TASK_TRACK_STARTED"] = config.get_boolean(
"POLYAXON_CELERY_TASK_TRACK_STARTED", is_optional=True, default=True)
context["CELERY_BROKER_POOL_LIMIT"] = config.get_int(
"POLYAXON_CELERY_BROKER_POOL_LIMIT", is_optional=True, default=100)
context["CELERY_BROKER_BACKEND"] = config.broker_backend
confirm_publish = config.get_boolean("POLYAXON_CELERY_CONFIRM_PUBLISH",
is_optional=True,
default=True)
context["CELERY_CONFIRM_PUBLISH"] = confirm_publish
if config.is_rabbitmq_broker and confirm_publish:
# see https://github.com/celery/celery/issues/5410 for details
context["CELERY_BROKER_TRANSPORT_OPTIONS"] = {"confirm_publish": True}
context["CELERY_BROKER_URL"] = config.get_broker_url()
context["INTERNAL_EXCHANGE"] = config.get_string(
"POLYAXON_INTERNAL_EXCHANGE", is_optional=True, default="internal")
result_bucked = config.get_string(
"POLYAXON_REDIS_CELERY_RESULT_BACKEND_URL",
is_optional=True,
)
if result_bucked:
context["CELERY_RESULT_BACKEND"] = config.get_redis_url(
"POLYAXON_REDIS_CELERY_RESULT_BACKEND_URL")
context["CELERY_WORKER_PREFETCH_MULTIPLIER"] = config.get_int(
"POLYAXON_CELERY_WORKER_PREFETCH_MULTIPLIER",
is_optional=True,
default=4)
eager_mode = config.get_boolean("POLYAXON_CELERY_TASK_ALWAYS_EAGER")
context["CELERY_TASK_ALWAYS_EAGER"] = eager_mode
if eager_mode:
context["CELERY_BROKER_TRANSPORT"] = "memory"
context["CELERY_ACCEPT_CONTENT"] = ["application/json"]
context["CELERY_TASK_SERIALIZER"] = "json"
context["CELERY_RESULT_SERIALIZER"] = "json"
context["CELERY_TASK_IGNORE_RESULT"] = True
context["CELERY_TIMEZONE"] = config.timezone
context["CELERY_HARD_TIME_LIMIT_DELAY"] = config.get_int(
"POLYAXON_CELERY_HARD_TIME_LIMIT_DELAY", is_optional=True, default=180)
context["CELERY_WORKER_MAX_TASKS_PER_CHILD"] = config.get_int(
"POLYAXON_CELERY_WORKER_MAX_TASKS_PER_CHILD",
is_optional=True,
default=100)
context["CELERY_WORKER_MAX_MEMORY_PER_CHILD"] = config.get_int(
"POLYAXON_CELERY_WORKER_MAX_MEMORY_PER_CHILD",
is_optional=True,
default=400000)
class Intervals:
"""All intervals are in seconds"""
OPERATIONS_DEFAULT_RETRY_DELAY = config.get_int(
"POLYAXON_INTERVALS_OPERATIONS_DEFAULT_RETRY_DELAY",
is_optional=True,
default=60,
)
OPERATIONS_MAX_RETRY_DELAY = config.get_int(
"POLYAXON_INTERVALS_OPERATIONS_MAX_RETRY_DELAY",
is_optional=True,
default=60 * 60,
)
RUNS_SCHEDULER = config.get_int("POLYAXON_INTERVALS_RUNS_SCHEDULER",
is_optional=True,
default=30)
context["Intervals"] = Intervals
context["CELERY_TASK_ROUTES"] = routes
def set_api(context, config: ConfigManager, processors: List[str] = None):
context["ROOT_URLCONF"] = "polyconf.urls"
platform_host = config.get_string("POLYAXON_PLATFORM_HOST",
is_optional=True)
context["PLATFORM_HOST"] = platform_host
def get_allowed_hosts():
allowed_hosts = config.get_string("POLYAXON_ALLOWED_HOSTS",
is_optional=True,
is_list=True,
default=["*"]) # type: list
if platform_host:
allowed_hosts.append(platform_host)
if ".polyaxon.com" not in allowed_hosts:
allowed_hosts.append(".polyaxon.com")
pod_ip = config.get_string("POLYAXON_POD_IP", is_optional=True)
if pod_ip:
allowed_hosts.append(pod_ip)
host_ip = config.get_string("POLYAXON_HOST_IP", is_optional=True)
if host_ip:
host_cidr = ".".join(host_ip.split(".")[:-1])
allowed_hosts += ["{}.{}".format(host_cidr, i) for i in range(255)]
return allowed_hosts
context["ALLOWED_HOSTS"] = get_allowed_hosts()
processors = processors or []
processors = [
"django.contrib.auth.context_processors.auth",
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.template.context_processors.media",
"django.template.context_processors.static",
"django.template.context_processors.tz",
"django.contrib.messages.context_processors.messages",
"polycommon.settings.context_processors.version",
"polycommon.settings.context_processors.assets_version",
"polycommon.settings.context_processors.ui_offline",
"polycommon.settings.context_processors.ui_enabled",
] + processors
context["FRONTEND_DEBUG"] = config.get_boolean("POLYAXON_FRONTEND_DEBUG")
template_debug = (config.get_boolean("DJANGO_TEMPLATE_DEBUG",
is_optional=True)
or config.is_debug_mode)
context["UI_ADMIN_ENABLED"] = config.get_boolean(
POLYAXON_KEYS_UI_ADMIN_ENABLED, is_optional=True, default=False)
context["UI_ASSETS_VERSION"] = config.get_string(
POLYAXON_KEYS_UI_ASSETS_VERSION, is_optional=True, default="")
context["UI_OFFLINE"] = config.get_boolean(POLYAXON_KEYS_UI_OFFLINE,
is_optional=True,
default=False)
context["UI_ENABLED"] = config.get_boolean(POLYAXON_KEYS_UI_ENABLED,
is_optional=True,
default=True)
context["TEMPLATES_DEBUG"] = template_debug
context["LIST_TEMPLATE_CONTEXT_PROCESSORS"] = processors
context["TEMPLATES"] = [{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"APP_DIRS": True,
"OPTIONS": {
"debug": template_debug,
"context_processors": processors
},
}]
def set_secrets(context, config: ConfigManager):
context["SECRET_KEY"] = config.get_string("POLYAXON_SECRET_KEY", is_secret=True)
context["SECRET_INTERNAL_TOKEN"] = config.get_string(
"POLYAXON_SECRET_INTERNAL_TOKEN", is_secret=True, is_optional=True
)
