def test_env_var_precedence_string():
set_flyte_config_file(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'configs/good.config'))
assert common.FlyteIntegerConfigurationEntry('madeup',
'int_value').get() == 3
assert common.FlyteStringConfigurationEntry('madeup',
'string_value').get() == 'abc'
old_environ = dict(os.environ)
try:
os.environ['FLYTE_MADEUP_INT_VALUE'] = '10'
os.environ["FLYTE_MADEUP_STRING_VALUE"] = 'overridden'
assert common.FlyteIntegerConfigurationEntry('madeup',
'int_value').get() == 10
assert common.FlyteStringConfigurationEntry(
'madeup', 'string_value').get() == 'overridden'
finally:
os.environ.clear()
os.environ.update(old_environ)
def test_env_var_precedence_string():
set_flyte_config_file(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
"configs/good.config"))
assert common.FlyteIntegerConfigurationEntry("madeup",
"int_value").get() == 3
assert common.FlyteStringConfigurationEntry("madeup",
"string_value").get() == "abc"
old_environ = dict(os.environ)
try:
os.environ["FLYTE_MADEUP_INT_VALUE"] = "10"
os.environ["FLYTE_MADEUP_STRING_VALUE"] = "overridden"
assert common.FlyteIntegerConfigurationEntry("madeup",
"int_value").get() == 10
assert common.FlyteStringConfigurationEntry(
"madeup", "string_value").get() == "overridden"
finally:
os.environ.clear()
os.environ.update(old_environ)
from __future__ import absolute_import
from flytekit.configuration import common as _config_common
S3_SHARD_FORMATTER = _config_common.FlyteRequiredStringConfigurationEntry(
'aws', 's3_shard_formatter')
S3_SHARD_STRING_LENGTH = _config_common.FlyteIntegerConfigurationEntry(
'aws', 's3_shard_string_length', default=2)
S3_ENDPOINT = _config_common.FlyteStringConfigurationEntry('aws',
'endpoint',
default=None)
S3_ACCESS_KEY_ID = _config_common.FlyteStringConfigurationEntry(
'aws', 'access_key_id', default=None)
S3_SECRET_ACCESS_KEY = _config_common.FlyteStringConfigurationEntry(
'aws', 'secret_access_key', default=None)
S3_ACCESS_KEY_ID_ENV_NAME = 'AWS_ACCESS_KEY_ID'
S3_SECRET_ACCESS_KEY_ENV_NAME = 'AWS_SECRET_ACCESS_KEY'
S3_ENDPOINT_ARG_NAME = '--endpoint-url'
"""
This is a Python format string which the SDK will use to generate names for workflows. Any template portion of the
string can only include 'module' or 'name'. So '{name}' is valid, but '{key}' is not. If not specified,
we fall back to the configuration for :py:attr:`flytekit.configuration.sdk.NAME_FORMAT`
"""
LAUNCH_PLAN_NAME_FORMAT = _config_common.FlyteStringConfigurationEntry(
"sdk", "launch_plan_name_format", fallback=NAME_FORMAT
)
"""
This is a Python format string which the SDK will use to generate names for launch plans. Any template portion of the
string can only include 'module' or 'name'. So '{name}' is valid, but '{key}' is not. If not specified,
we fall back to the configuration for :py:attr:`flytekit.configuration.sdk.NAME_FORMAT`
"""
LOGGING_LEVEL = _config_common.FlyteIntegerConfigurationEntry("sdk", "logging_level", default=20)
"""
This is the default logging level for the Python logging library and will be set before user code runs.
Note that this configuration is special in that it is a runtime setting, not a compile time setting. This is the only
runtime option in this file.
"""
PARQUET_ENGINE = _config_common.FlyteStringConfigurationEntry("sdk", "parquet_engine", default="pyarrow")
"""
This is the parquet engine to use when reading data from parquet files.
"""
FAST_REGISTRATION_DIR = _config_common.FlyteStringConfigurationEntry("sdk", "fast_registration_dir")
"""
This is the remote directory where fast-registered code will be uploaded to.
Users calling fast-execute need write permission to this directory.
from __future__ import absolute_import
from flytekit.configuration import common as _common_config
HOST = _common_config.FlyteStringConfigurationEntry('statsd', 'host', default='localhost')
PORT = _common_config.FlyteIntegerConfigurationEntry('statsd', 'port', default=8125)
we fall back to the configuration for :py:attr:`flytekit.configuration.sdk.NAME_FORMAT`
"""
WORKFLOW_NAME_FORMAT = _config_common.FlyteStringConfigurationEntry('sdk', 'workflow_name_format', fallback=NAME_FORMAT)
"""
This is a Python format string which the SDK will use to generate names for workflows. Any template portion of the
string can only include 'module' or 'name'. So '{name}' is valid, but '{key}' is not. If not specified,
we fall back to the configuration for :py:attr:`flytekit.configuration.sdk.NAME_FORMAT`
"""
LAUNCH_PLAN_NAME_FORMAT = _config_common.FlyteStringConfigurationEntry(
'sdk', 'launch_plan_name_format', fallback=NAME_FORMAT
)
"""
This is a Python format string which the SDK will use to generate names for launch plans. Any template portion of the
string can only include 'module' or 'name'. So '{name}' is valid, but '{key}' is not. If not specified,
we fall back to the configuration for :py:attr:`flytekit.configuration.sdk.NAME_FORMAT`
"""
LOGGING_LEVEL = _config_common.FlyteIntegerConfigurationEntry('sdk', 'logging_level', default=20)
"""
This is the default logging level for the Python logging library and will be set before user code runs.
Note that this configuration is special in that it is a runtime setting, not a compile time setting. This is the only
runtime option in this file.
"""
PARQUET_ENGINE = _config_common.FlyteStringConfigurationEntry('sdk', 'parquet_engine', default='pyarrow')
"""
This is the parquet engine to use when reading data from parquet files.
"""
from flytekit.configuration import common as _config_common
S3_SHARD_FORMATTER = _config_common.FlyteRequiredStringConfigurationEntry(
"aws", "s3_shard_formatter")
S3_SHARD_STRING_LENGTH = _config_common.FlyteIntegerConfigurationEntry(
"aws", "s3_shard_string_length", default=2)
S3_ENDPOINT = _config_common.FlyteStringConfigurationEntry("aws",
"endpoint",
default=None)
S3_ACCESS_KEY_ID = _config_common.FlyteStringConfigurationEntry(
"aws", "access_key_id", default=None)
S3_SECRET_ACCESS_KEY = _config_common.FlyteStringConfigurationEntry(
"aws", "secret_access_key", default=None)
S3_ACCESS_KEY_ID_ENV_NAME = "AWS_ACCESS_KEY_ID"
S3_SECRET_ACCESS_KEY_ENV_NAME = "AWS_SECRET_ACCESS_KEY"
S3_ENDPOINT_ARG_NAME = "--endpoint-url"
ENABLE_DEBUG = _config_common.FlyteBoolConfigurationEntry("aws",
"enable_debug",
default=False)
RETRIES = _config_common.FlyteIntegerConfigurationEntry("aws",
"retries",
default=3)
# Execution project and domain represent the values passed by execution engine at runtime.
EXECUTION_PROJECT = _common_config.FlyteStringConfigurationEntry(
'internal', 'execution_project', default="")
EXECUTION_DOMAIN = _common_config.FlyteStringConfigurationEntry(
'internal', 'execution_domain', default="")
EXECUTION_WORKFLOW = _common_config.FlyteStringConfigurationEntry(
'internal', 'execution_workflow', default="")
EXECUTION_LAUNCHPLAN = _common_config.FlyteStringConfigurationEntry(
'internal', 'execution_launchplan', default="")
EXECUTION_NAME = _common_config.FlyteStringConfigurationEntry('internal',
'execution_id',
default="")
# This is another layer of logging level, which can be set by propeller, and can override the SDK configuration if
# necessary. (See the sdk.py version of this as well.)
LOGGING_LEVEL = _common_config.FlyteIntegerConfigurationEntry(
'internal', 'logging_level')
_IMAGE_VERSION_REGEX = '.*:(.+)'
def look_up_version_from_image_tag(tag):
"""
Looks up the image tag from environment variable (should be set from the Dockerfile).
FLYTE_INTERNAL_IMAGE should be the environment variable.
This function is used when registering tasks/workflows with Admin.
When using the canonical Python-based development cycle, the version that is used to register workflows
and tasks with Admin should be the version of the image itself, which should ideally be something unique
like the sha of the latest commit.
:param Text tag: e.g. somedocker.com/myimage:someversion123
# Execution project and domain represent the values passed by execution engine at runtime.
EXECUTION_PROJECT = _common_config.FlyteStringConfigurationEntry(
"internal", "execution_project", default="")
EXECUTION_DOMAIN = _common_config.FlyteStringConfigurationEntry(
"internal", "execution_domain", default="")
EXECUTION_WORKFLOW = _common_config.FlyteStringConfigurationEntry(
"internal", "execution_workflow", default="")
EXECUTION_LAUNCHPLAN = _common_config.FlyteStringConfigurationEntry(
"internal", "execution_launchplan", default="")
EXECUTION_NAME = _common_config.FlyteStringConfigurationEntry("internal",
"execution_id",
default="")
# This is another layer of logging level, which can be set by propeller, and can override the SDK configuration if
# necessary. (See the sdk.py version of this as well.)
LOGGING_LEVEL = _common_config.FlyteIntegerConfigurationEntry(
"internal", "logging_level")
_IMAGE_VERSION_REGEX = ".*:(.+)"
def look_up_version_from_image_tag(tag):
"""
Looks up the image tag from environment variable (should be set from the Dockerfile).
FLYTE_INTERNAL_IMAGE should be the environment variable.
This function is used when registering tasks/workflows with Admin.
When using the canonical Python-based development cycle, the version that is used to register workflows
and tasks with Admin should be the version of the image itself, which should ideally be something unique
like the sha of the latest commit.
:param Text tag: e.g. somedocker.com/myimage:someversion123
from flytekit.configuration import common as _common_config
HOST = _common_config.FlyteStringConfigurationEntry("statsd",
"host",
default="localhost")
PORT = _common_config.FlyteIntegerConfigurationEntry("statsd",
"port",
default=8125)
DISABLED = _common_config.FlyteBoolConfigurationEntry("statsd",
"disabled",
default=False)