def test_get_broker(self): os.environ["POLYAXON_ENVIRONMENT"] = "testing" os.environ.pop("POLYAXON_BROKER_BACKEND", None) config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json" ]) assert config.broker_backend == "rabbitmq" assert config.is_redis_broker is False assert config.is_rabbitmq_broker is True config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_BROKER_BACKEND": "rabbitmq" }, ]) assert config.broker_backend == "rabbitmq" config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_BROKER_BACKEND": "redis" }, ]) assert config.broker_backend == "redis" assert config.is_redis_broker is True assert config.is_rabbitmq_broker is False
def set_cors(context, config: ConfigManager): # session settings context["CORS_ALLOW_CREDENTIALS"] = True whitelist = config.get_list("POLYAXON_CORS_ORIGIN_WHITELIST", is_optional=True, default=[]) context["CORS_ORIGIN_WHITELIST"] = whitelist context["CORS_ORIGIN_ALLOW_ALL"] = False if whitelist else True context["CORS_ALLOW_HEADERS"] = default_headers + ( PolyaxonServiceHeaders.CLI_VERSION, PolyaxonServiceHeaders.CLIENT_VERSION, PolyaxonServiceHeaders.INTERNAL, PolyaxonServiceHeaders.SERVICE, ) ssl_enabled = config.get_boolean("POLYAXON_SSL_ENABLED", is_optional=True, default=False) ssl_redirect_enabled = config.get_boolean("POLYAXON_SSL_REDIRECT_ENABLED", is_optional=True, default=False) context["SSL_ENABLED"] = ssl_enabled context["PROTOCOL"] = "http" context["WS_PROTOCOL"] = "ws" if ssl_enabled: context["SESSION_COOKIE_SECURE"] = True context["CSRF_COOKIE_SECURE"] = True context["SECURE_PROXY_SSL_HEADER"] = ("HTTP_X_FORWARDED_PROTO", "https") context["PROTOCOL"] = "https" context["WS_PROTOCOL"] = "wss" if ssl_redirect_enabled: context["SECURE_SSL_REDIRECT"] = True
def set_encryption(context, config: ConfigManager): context["ENCRYPTION_KEY"] = config.get_string("POLYAXON_ENCRYPTION_KEY", is_optional=True) context["ENCRYPTION_SECRET"] = config.get_string( "POLYAXON_ENCRYPTION_SECRET", is_optional=True) context["ENCRYPTION_BACKEND"] = config.get_string( "POLYAXON_ENCRYPTION_BACKEND", is_optional=True)
def set_cors(context, config: ConfigManager): # session settings context["CORS_ALLOW_CREDENTIALS"] = True allowed_list = config.get_list("POLYAXON_CORS_ALLOWED_ORIGINS", is_optional=True, default=[]) context["CORS_ALLOWED_ORIGINS"] = allowed_list context["CORS_ALLOW_ALL_ORIGINS"] = False if allowed_list else True context["CORS_ALLOW_HEADERS"] = (default_headers + PolyaxonServiceHeaders.get_headers()) ssl_enabled = config.get_boolean("POLYAXON_SSL_ENABLED", is_optional=True, default=False) ssl_redirect_enabled = config.get_boolean("POLYAXON_SSL_REDIRECT_ENABLED", is_optional=True, default=False) context["SSL_ENABLED"] = ssl_enabled context["PROTOCOL"] = "http" context["WS_PROTOCOL"] = "ws" if ssl_enabled: context["SESSION_COOKIE_SECURE"] = True context["CSRF_COOKIE_SECURE"] = True context["SECURE_PROXY_SSL_HEADER"] = ("HTTP_X_FORWARDED_PROTO", "https") context["PROTOCOL"] = "https" context["WS_PROTOCOL"] = "wss" if ssl_redirect_enabled: context["SECURE_SSL_REDIRECT"] = True
def set_api(context, config: ConfigManager, processors: List[str] = None): context["ROOT_URLCONF"] = "polyconf.urls" def get_allowed_hosts(): allowed_hosts = config.get_string("POLYAXON_ALLOWED_HOSTS", is_optional=True, is_list=True, default=["*"]) allowed_hosts.append(".polyaxon.com") platform_host = config.get_string("POLYAXON_PLATFORM_HOST", is_optional=True) if platform_host: allowed_hosts.append(platform_host) return allowed_hosts context["ALLOWED_HOSTS"] = get_allowed_hosts() processors = processors or [] processors = [ "django.contrib.auth.context_processors.auth", "django.template.context_processors.debug", "django.template.context_processors.media", "django.template.context_processors.static", "django.template.context_processors.tz", "django.contrib.messages.context_processors.messages", "polycommon.settings.context_processors.versions", "polycommon.settings.context_processors.ui_offline", "polycommon.settings.context_processors.ui_enabled", ] + processors context["FRONTEND_DEBUG"] = config.get_boolean("POLYAXON_FRONTEND_DEBUG") template_debug = (config.get_boolean("DJANGO_TEMPLATE_DEBUG", is_optional=True) or config.is_debug_mode) context["UI_ADMIN_ENABLED"] = config.get_boolean( POLYAXON_KEYS_UI_ADMIN_ENABLED, is_optional=True, default=False) context["UI_OFFLINE"] = config.get_boolean(POLYAXON_KEYS_UI_OFFLINE, is_optional=True, default=False) context["UI_ENABLED"] = config.get_boolean(POLYAXON_KEYS_UI_ENABLED, is_optional=True, default=True) context["TEMPLATES_DEBUG"] = template_debug context["LIST_TEMPLATE_CONTEXT_PROCESSORS"] = processors context["TEMPLATES"] = [{ "BACKEND": "django.template.backends.django.DjangoTemplates", "APP_DIRS": True, "OPTIONS": { "debug": template_debug, "context_processors": processors }, }]
def set_apps( context, config: ConfigManager, default_apps: Tuple, third_party_apps: Optional[Tuple], project_apps: Tuple, ): extra_apps = config.get_string( "POLYAXON_EXTRA_APPS", is_list=True, is_optional=True ) extra_apps = tuple(extra_apps) if extra_apps else () apps = ( "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", ) third_party_apps = third_party_apps or () context["INSTALLED_APPS"] = ( apps + third_party_apps + default_apps + extra_apps + project_apps )
def set_admin(context, config: ConfigManager): admin_name = config.get_string("POLYAXON_ADMIN_NAME", is_optional=True) admin_mail = config.get_string("POLYAXON_ADMIN_MAIL", is_optional=True) if admin_mail and admin_mail: admins = ((admin_name, admin_mail), ) context["ADMINS"] = admins context["MANAGERS"] = admins
def set_assets(context, root_dir, config: ConfigManager): context["MEDIA_ROOT"] = config.get_string("POLYAXON_MEDIA_ROOT") context["MEDIA_URL"] = config.get_string("POLYAXON_MEDIA_URL") context["STATIC_ROOT"] = config.get_string("POLYAXON_STATIC_ROOT") context["STATIC_URL"] = config.get_string("POLYAXON_STATIC_URL") # Additional locations of static files context["STATICFILES_DIRS"] = (str(root_dir.parent.parent / "public"), ) context["STATICFILES_FINDERS"] = ( "django.contrib.staticfiles.finders.FileSystemFinder", "django.contrib.staticfiles.finders.AppDirectoriesFinder", ) context["LOCALE_PATHS"] = ( str(root_dir / "locale"), str(root_dir / "client" / "js" / "libs" / "locale"), ) context["STATICI18N_ROOT"] = "static" context["STATICI18N_OUTPUT_DIR"] = "jsi18n" context["ARTIFACTS_ROOT"] = config.get_string( "POLYAXON_ARTIFACTS_ROOT", is_optional=True, default="/tmp/plx/artifacts_uploads", ) context["LOGS_ROOT"] = config.get_string("POLYAXON_LOGS_ROOT", is_optional=True, default="/tmp/plx/logs_uploads") context["ARCHIVES_ROOT"] = config.get_string("POLYAXON_ARCHIVES_ROOT", is_optional=True, default="/tmp/plx/archives")
def test_get_from_os_env(self): os.environ["POLYAXON_ENVIRONMENT"] = "testing" os.environ["FOO_BAR_KEY"] = "foo_bar" config = ConfigManager.read_configs( [os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json"] ) assert config.get_string("POLYAXON_ENVIRONMENT") == "testing" assert config.get_string("FOO_BAR_KEY") == "foo_bar"
def set_core(context, config: ConfigManager): context["DEBUG"] = config.is_debug_mode context["POLYAXON_SERVICE"] = config.service context["POLYAXON_ENVIRONMENT"] = config.env context["CHART_VERSION"] = config.chart_version context["SCHEDULER_ENABLED"] = config.scheduler_enabled context["K8S_NAMESPACE"] = config.namespace context["FILE_UPLOAD_PERMISSIONS"] = 0o644 context["WSGI_APPLICATION"] = "polyconf.wsgi.application" context["TIME_ZONE"] = config.timezone context["LANGUAGE_CODE"] = "en" context["LANGUAGES"] = (("en", u"English"), ) context["USE_I18N"] = True context["USE_L10N"] = True context["USE_TZ"] = True context["INTERNAL_IPS"] = ("127.0.0.1", ) context["APPEND_SLASH"] = True context["ROOT_URLCONF"] = "" db_engine = "django.db.backends.postgresql" context["DEFAULT_DB_ENGINE"] = db_engine db_definition = { "ENGINE": config.get_string("POLYAXON_DB_ENGINE", is_optional=True, default=db_engine), "NAME": config.get_string("POLYAXON_DB_NAME"), "USER": config.get_string("POLYAXON_DB_USER"), "PASSWORD": config.get_string("POLYAXON_DB_PASSWORD", is_secret=True), "HOST": config.get_string("POLYAXON_DB_HOST"), "PORT": config.get_string("POLYAXON_DB_PORT"), "ATOMIC_REQUESTS": True, "CONN_MAX_AGE": config.get_int("POLYAXON_DB_CONN_MAX_AGE", is_optional=True, default=None), } db_options = config.get_dict("POLYAXON_DB_OPTIONS", is_optional=True, default={}) if db_options: db_definition["OPTIONS"] = db_options context["DATABASES"] = {"default": db_definition}
def test_get_broker_url(self): os.environ["POLYAXON_ENVIRONMENT"] = "testing" os.environ.pop("POLYAXON_RABBITMQ_USER", None) os.environ.pop("POLYAXON_RABBITMQ_PASSWORD", None) config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_BROKER_BACKEND": "redis", "POLYAXON_REDIS_CELERY_BROKER_URL": "foo", }, ]) assert config.get_broker_url() == "redis://foo" config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_REDIS_PROTOCOL": "rediss", "POLYAXON_BROKER_BACKEND": "redis", "POLYAXON_REDIS_CELERY_BROKER_URL": "foo", }, ]) assert config.get_broker_url() == "rediss://foo" config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_BROKER_BACKEND": "redis", "POLYAXON_REDIS_CELERY_BROKER_URL": "foo", "POLYAXON_REDIS_PASSWORD": "******", }, ]) assert config.get_broker_url() == "redis://:pass@foo" config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_AMQP_URL": "foo", "POLYAXON_BROKER_BACKEND": "rabbitmq", "POLYAXON_REDIS_CELERY_BROKER_URL": "foo", }, ]) assert config.get_broker_url() == "amqp://foo" config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_AMQP_URL": "foo", "POLYAXON_BROKER_BACKEND": "rabbitmq", "POLYAXON_RABBITMQ_PASSWORD": "", "POLYAXON_REDIS_CELERY_BROKER_URL": "foo", }, ]) assert config.get_broker_url() == "amqp://foo" config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_AMQP_URL": "foo", "POLYAXON_BROKER_BACKEND": "rabbitmq", "POLYAXON_RABBITMQ_PASSWORD": "", "POLYAXON_RABBITMQ_USER": "******", }, ]) assert config.get_broker_url() == "amqp://foo" config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_AMQP_URL": "foo", "POLYAXON_BROKER_BACKEND": "rabbitmq", "POLYAXON_RABBITMQ_USER": "", "POLYAXON_RABBITMQ_PASSWORD": "******", }, ]) assert config.get_broker_url() == "amqp://foo" config = ConfigManager.read_configs([ os.environ, "./tests/fixtures_static/configs/non_opt_config_tests.json", { "POLYAXON_AMQP_URL": "foo", "POLYAXON_BROKER_BACKEND": "rabbitmq", "POLYAXON_RABBITMQ_USER": "******", "POLYAXON_RABBITMQ_PASSWORD": "******", }, ]) assert config.get_broker_url() == "amqp://*****:*****@foo"
def test_raises_for_non_optional_env_vars(self): with self.assertRaises(PolyaxonSchemaError): ConfigManager.read_configs([os.environ])
def set_celery(context, config: ConfigManager, routes: Dict): context["CELERY_TASK_TRACK_STARTED"] = config.get_boolean( "POLYAXON_CELERY_TASK_TRACK_STARTED", is_optional=True, default=True) context["CELERY_BROKER_POOL_LIMIT"] = config.get_int( "POLYAXON_CELERY_BROKER_POOL_LIMIT", is_optional=True, default=100) context["CELERY_BROKER_BACKEND"] = config.broker_backend confirm_publish = config.get_boolean("POLYAXON_CELERY_CONFIRM_PUBLISH", is_optional=True, default=True) context["CELERY_CONFIRM_PUBLISH"] = confirm_publish if config.is_rabbitmq_broker and confirm_publish: # see https://github.com/celery/celery/issues/5410 for details context["CELERY_BROKER_TRANSPORT_OPTIONS"] = {"confirm_publish": True} context["CELERY_BROKER_URL"] = config.get_broker_url() context["INTERNAL_EXCHANGE"] = config.get_string( "POLYAXON_INTERNAL_EXCHANGE", is_optional=True, default="internal") result_bucked = config.get_string( "POLYAXON_REDIS_CELERY_RESULT_BACKEND_URL", is_optional=True, ) if result_bucked: context["CELERY_RESULT_BACKEND"] = config.get_redis_url( "POLYAXON_REDIS_CELERY_RESULT_BACKEND_URL") context["CELERY_WORKER_PREFETCH_MULTIPLIER"] = config.get_int( "POLYAXON_CELERY_WORKER_PREFETCH_MULTIPLIER", is_optional=True, default=4) eager_mode = config.get_boolean("POLYAXON_CELERY_TASK_ALWAYS_EAGER") context["CELERY_TASK_ALWAYS_EAGER"] = eager_mode if eager_mode: context["CELERY_BROKER_TRANSPORT"] = "memory" context["CELERY_ACCEPT_CONTENT"] = ["application/json"] context["CELERY_TASK_SERIALIZER"] = "json" context["CELERY_RESULT_SERIALIZER"] = "json" context["CELERY_TASK_IGNORE_RESULT"] = True context["CELERY_TIMEZONE"] = config.timezone context["CELERY_HARD_TIME_LIMIT_DELAY"] = config.get_int( "POLYAXON_CELERY_HARD_TIME_LIMIT_DELAY", is_optional=True, default=180) context["CELERY_WORKER_MAX_TASKS_PER_CHILD"] = config.get_int( "POLYAXON_CELERY_WORKER_MAX_TASKS_PER_CHILD", is_optional=True, default=100) context["CELERY_WORKER_MAX_MEMORY_PER_CHILD"] = config.get_int( "POLYAXON_CELERY_WORKER_MAX_MEMORY_PER_CHILD", is_optional=True, default=400000) class Intervals: """All intervals are in seconds""" OPERATIONS_DEFAULT_RETRY_DELAY = config.get_int( "POLYAXON_INTERVALS_OPERATIONS_DEFAULT_RETRY_DELAY", is_optional=True, default=60, ) OPERATIONS_MAX_RETRY_DELAY = config.get_int( "POLYAXON_INTERVALS_OPERATIONS_MAX_RETRY_DELAY", is_optional=True, default=60 * 60, ) RUNS_SCHEDULER = config.get_int("POLYAXON_INTERVALS_RUNS_SCHEDULER", is_optional=True, default=30) context["Intervals"] = Intervals context["CELERY_TASK_ROUTES"] = routes
def set_api(context, config: ConfigManager, processors: List[str] = None): context["ROOT_URLCONF"] = "polyconf.urls" platform_host = config.get_string("POLYAXON_PLATFORM_HOST", is_optional=True) context["PLATFORM_HOST"] = platform_host def get_allowed_hosts(): allowed_hosts = config.get_string("POLYAXON_ALLOWED_HOSTS", is_optional=True, is_list=True, default=["*"]) # type: list if platform_host: allowed_hosts.append(platform_host) if ".polyaxon.com" not in allowed_hosts: allowed_hosts.append(".polyaxon.com") pod_ip = config.get_string("POLYAXON_POD_IP", is_optional=True) if pod_ip: allowed_hosts.append(pod_ip) host_ip = config.get_string("POLYAXON_HOST_IP", is_optional=True) if host_ip: host_cidr = ".".join(host_ip.split(".")[:-1]) allowed_hosts += ["{}.{}".format(host_cidr, i) for i in range(255)] return allowed_hosts context["ALLOWED_HOSTS"] = get_allowed_hosts() processors = processors or [] processors = [ "django.contrib.auth.context_processors.auth", "django.template.context_processors.debug", "django.template.context_processors.request", "django.template.context_processors.media", "django.template.context_processors.static", "django.template.context_processors.tz", "django.contrib.messages.context_processors.messages", "polycommon.settings.context_processors.version", "polycommon.settings.context_processors.assets_version", "polycommon.settings.context_processors.ui_offline", "polycommon.settings.context_processors.ui_enabled", ] + processors context["FRONTEND_DEBUG"] = config.get_boolean("POLYAXON_FRONTEND_DEBUG") template_debug = (config.get_boolean("DJANGO_TEMPLATE_DEBUG", is_optional=True) or config.is_debug_mode) context["UI_ADMIN_ENABLED"] = config.get_boolean( POLYAXON_KEYS_UI_ADMIN_ENABLED, is_optional=True, default=False) context["UI_ASSETS_VERSION"] = config.get_string( POLYAXON_KEYS_UI_ASSETS_VERSION, is_optional=True, default="") context["UI_OFFLINE"] = config.get_boolean(POLYAXON_KEYS_UI_OFFLINE, is_optional=True, default=False) context["UI_ENABLED"] = config.get_boolean(POLYAXON_KEYS_UI_ENABLED, is_optional=True, default=True) context["TEMPLATES_DEBUG"] = template_debug context["LIST_TEMPLATE_CONTEXT_PROCESSORS"] = processors context["TEMPLATES"] = [{ "BACKEND": "django.template.backends.django.DjangoTemplates", "APP_DIRS": True, "OPTIONS": { "debug": template_debug, "context_processors": processors }, }]
def set_secrets(context, config: ConfigManager): context["SECRET_KEY"] = config.get_string("POLYAXON_SECRET_KEY", is_secret=True) context["SECRET_INTERNAL_TOKEN"] = config.get_string( "POLYAXON_SECRET_INTERNAL_TOKEN", is_secret=True, is_optional=True )