def _build_gcs_values(
self,
_env_file: Union[Path, str, None] = None,
_env_file_encoding: Optional[str] = None
) -> Dict[str, Optional[str]]:
if self.__config__.case_sensitive:
env_vars: Mapping[str, Optional[str]] = os.environ
else:
env_vars = {k.lower(): v for k, v in os.environ.items()}
env_file = _env_file if _env_file != env_file_sentinel else self.__config__.env_file
env_file_encoding = _env_file_encoding if _env_file_encoding is not None else self.__config__.env_file_encoding
if env_file is not None:
env_path = Path(env_file).expanduser()
if env_path.is_file():
env_vars = {
**read_env_file(env_path,
encoding=env_file_encoding,
case_sensitive=self.__config__.case_sensitive),
**env_vars,
}
env = dict()
for field in self.__fields__.values():
cloud_key = field.field_info.extra.get("cloud_key")
if not isinstance(cloud_key, str):
continue
cloud_key = cloud_key.lower(
) if not self.__config__.case_sensitive else cloud_key
if not env_vars.get(cloud_key):
continue
env[field.alias] = get_google_cloud_secret(env_vars[cloud_key])
return env
def _build_environ(
self,
_env_file: Union[Path, str, None] = None,
_env_file_encoding: Optional[str] = None
) -> Dict[str, Optional[str]]:
"""
Build environment variables suitable for passing to the Model.
"""
d: Dict[str, Optional[str]] = {}
if self.__config__.case_sensitive:
env_vars: Mapping[str, Optional[str]] = os.environ
else:
env_vars = {k.lower(): v for k, v in os.environ.items()}
env_file_vars: Dict[str, Optional[str]] = {}
env_file = _env_file if _env_file != env_file_sentinel else self.__config__.env_file
env_file_encoding = _env_file_encoding if _env_file_encoding is not None else self.__config__.env_file_encoding
if env_file is not None:
env_path = Path(env_file)
if env_path.is_file():
env_file_vars = read_env_file(
env_path,
encoding=env_file_encoding,
case_sensitive=self.__config__.case_sensitive)
env_vars = {**env_file_vars, **env_vars}
for field in self.__fields__.values():
env_val: Optional[str] = None
for env_name in field.field_info.extra['env_names']:
env_val = env_vars.get(env_name)
if env_name in env_file_vars:
del env_file_vars[env_name]
if env_val is not None:
break
if env_val is None:
continue
if field.is_complex():
try:
env_val = self.__config__.json_loads(env_val)
except ValueError as e:
raise SettingsError(
f'error parsing JSON for "{env_name}"' # type: ignore
) from e
d[field.alias] = env_val
if env_file_vars:
for env_name, env_val in env_file_vars.items():
try:
env_val = self.__config__.json_loads(env_val)
except ValueError as e:
pass
d[env_name] = env_val
return d
def test_read_env_file_syntax_wrong(tmp_path):
p = tmp_path / '.env'
p.write_text('NOT_AN_ASSIGNMENT')
assert read_env_file(p, case_sensitive=True) == {'NOT_AN_ASSIGNMENT': None}
def test_read_env_file_cast_sensitive(tmp_path):
p = tmp_path / '.env'
p.write_text('a="test"\nB=123')
assert read_env_file(p) == {'a': 'test', 'b': '123'}
assert read_env_file(p, case_sensitive=True) == {'a': 'test', 'B': '123'}
def __call__(self, settings: BaseSettings) -> Dict[str, Any]:
"""
Build environment variables suitable for passing to the Model.
"""
d: Dict[str, Optional[str]] = {}
if settings.__config__.case_sensitive:
env_vars: Mapping[str,
Optional[str]] = os.environ # pragma: no cover
else:
env_vars = {k.lower(): v for k, v in os.environ.items()}
env_file_vars: Dict[str, Optional[str]] = {}
env_file = (self.env_file if self.env_file != env_file_sentinel else
settings.__config__.env_file)
env_file_encoding = (self.env_file_encoding
if self.env_file_encoding is not None else
settings.__config__.env_file_encoding)
if env_file is not None:
env_path = Path(env_file)
if env_path.is_file():
env_file_vars = read_env_file(
env_path,
encoding=env_file_encoding,
case_sensitive=settings.__config__.case_sensitive,
)
env_vars = {**env_file_vars, **env_vars}
for field in settings.__fields__.values():
env_val: Optional[str] = None
for env_name in field.field_info.extra["env_names"]:
env_val = env_vars.get(env_name)
if env_name in env_file_vars:
del env_file_vars[env_name]
if env_val is not None:
break
if env_val is None:
continue
if field.is_complex():
try:
env_val = settings.__config__.json_loads(env_val)
except ValueError as e: # pragma: no cover
raise SettingsError(
f'error parsing JSON for "{env_name}"' # type: ignore
) from e
d[field.alias] = env_val
if env_file_vars:
for env_name, env_val in env_file_vars.items():
if (env_val is None
or len(env_val) == 0) and env_name in env_vars:
env_val = env_vars[env_name]
try:
if env_val:
env_val = settings.__config__.json_loads(
env_val.strip())
except ValueError as e:
logger.opt(colors=True, exception=e).trace(
f"Error while parsing JSON for {escape_tag(env_name)}. Assumed as string."
)
d[env_name] = env_val
return d
from pydantic import BaseModel, BaseSettings, Field
from typing import List, Union, Optional, Dict
from enum import Enum
from .config import ConfigDefault
logger = logging.getLogger("vaex.settings")
_default_home = vaex.utils.get_vaex_home()
try:
import dotenv
has_dotenv = True
except:
has_dotenv = False
if has_dotenv:
from pydantic.env_settings import read_env_file
envs = read_env_file(ConfigDefault.env_file)
_default_home = envs.get('vaex_home', _default_home)
# we may want to use this
# class ByteAmount(str):
# @classmethod
# def validate(cls, value):
# try:
# import ast
# value = ast.literal_eval(value)
# except:
# pass
# if isinstance(value, str):
# import dask
# value = dask.utils.parse_bytes(value)
# if not isinstance(value, int):
