def _parse_params(path_params): from ruamel.yaml import YAMLError from dvc.dependency.param import ParamsDependency from dvc.utils.flatten import unflatten from dvc.utils.serialize import loads_yaml ret = {} for path_param in path_params: path, _, params_str = path_param.rpartition(":") # remove empty strings from params, on condition such as `-p "file1:"` params = {} for param_str in filter(bool, params_str.split(",")): try: # interpret value strings using YAML rules key, value = param_str.split("=") params[key] = loads_yaml(value) except (ValueError, YAMLError): raise InvalidArgumentError( f"Invalid param/value pair '{param_str}'" ) if not path: path = ParamsDependency.DEFAULT_PARAMS_FILE ret[path] = unflatten(params) return ret
def _prepare_credentials(self, **config): from dvc.config import ConfigError from dvc.utils.flatten import flatten, unflatten login_info = defaultdict(dict) # credentials login_info["key"] = config.get("access_key_id") login_info["secret"] = config.get("secret_access_key") login_info["token"] = config.get("session_token") # session configuration login_info["profile"] = config.get("profile") login_info["use_ssl"] = config.get("use_ssl", True) # extra client configuration client = login_info["client_kwargs"] client["region_name"] = config.get("region") client["endpoint_url"] = config.get("endpointurl") client["verify"] = config.get("ssl_verify") # encryptions additional = login_info["s3_additional_kwargs"] additional["ServerSideEncryption"] = config.get("sse") additional["SSEKMSKeyId"] = config.get("sse_kms_key_id") additional["ACL"] = config.get("acl") for grant_option, grant_key in self._GRANTS.items(): if config.get(grant_option): if additional["ACL"]: raise ConfigError( "`acl` and `grant_*` AWS S3 config options " "are mutually exclusive") additional[grant_key] = config[grant_option] # config kwargs session_config = login_info["config_kwargs"] session_config["s3"] = self._load_aws_config_file( login_info["profile"]) shared_creds = config.get("credentialpath") if shared_creds: os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) if (client["region_name"] is None and session_config["s3"].get("region_name") is None and os.getenv("AWS_REGION") is None): # Enable bucket region caching login_info["cache_regions"] = config.get("cache_regions", True) config_path = config.get("configpath") if config_path: os.environ.setdefault("AWS_CONFIG_FILE", config_path) return unflatten({ key: value for key, value in flatten(login_info).items() if value is not None })