def _load_file(cls: Type[T], path: Path) -> T:
if not path.exists():
return cls()
with path.open() as f:
try:
config = toml.load(f)
except ValueError as e:
raise ConfigurationError(f"TOML file {path} is unparasble: {e}") from e
# In previous versions of remote, `include_vcs_ignore_patterns` key was named with a typo
# Now we need to check if config is using the old name to maintain backward compatibility
_backward_compatible_sanitize(config, {"include_vsc_ignore_patterns": "include_vcs_ignore_patterns"})
try:
return cls.parse_obj(config)
except ValidationError as e:
messages = []
for err in e.errors():
location = ".".join((str(x) for x in err["loc"]))
reason = err["msg"]
messages.append(f" - {location}: {reason}")
msg = "\n".join(messages)
raise ConfigurationError(f"Invalid value in configuration file {path}:\n{msg}") from e
def _extract_shell_info(line: str, env_vars: List[str]) -> Tuple[str, str]:
if not env_vars:
return DEFAULT_SHELL, DEFAULT_SHELL_OPTIONS
vars_string = env_vars[0]
env = {}
items = vars_string.split()
index = 0
while index < len(items):
key, value = items[index].split("=")
if value.startswith("'") or value.startswith('"'):
control_character = value[0]
while index < len(items) - 1:
if value[-1] == control_character:
break
index += 1
value += " " + items[index]
if not value[-1] == control_character:
raise ConfigurationError(
f"Config line {line} is corrupted. Cannot parse end {key}={value}"
)
env[key] = value.strip("\"'")
index += 1
print(env)
# TODO: these shell types are not used in new implementation, need to remove them
shell = env.pop("RSHELL", DEFAULT_SHELL)
shell_options = env.pop("RSHELL_OPTS", DEFAULT_SHELL_OPTIONS)
if env:
raise ConfigurationError(
f"Config line {line} contains unexpected env variables: {env}. Only RSHELL and RSHELL_OPTS can be used"
)
return shell, shell_options
def load_ignores(workspace_root: Path) -> SyncRules:
ignores: Dict[str, List[str]] = defaultdict(list)
ignores["both"].extend(BASE_IGNORES)
ignore_file = workspace_root / IGNORE_FILE_NAME
if not ignore_file.exists():
return _postprocess(ignores)
active_section = "both"
is_new_format = None
for line in ignore_file.read_text().splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
matcher = IGNORE_SECTION_REGEX.match(line)
if matcher is None:
if is_new_format is None:
is_new_format = False
ignores[active_section].append(line)
else:
if is_new_format is None:
is_new_format = True
elif not is_new_format:
raise ConfigurationError(
f"Few ignore patters were listed in {IGNORE_FILE_NAME} before the first section {matcher.group(1)} appeared. "
"Please list all ignored files after a section declaration if you use new ignore format"
)
active_section = matcher.group(1)
return _postprocess(ignores)
def _postprocess(ignores):
pull = ignores.pop("pull", [])
push = ignores.pop("push", [])
both = ignores.pop("both", [])
if ignores:
raise ConfigurationError(
f"{IGNORE_FILE_NAME} file has unexpected sections: {', '.join(ignores.keys())}. Please remove them"
)
return SyncRules(pull=pull, push=push, both=both)
def resolve_workspace_root(
working_dir: Path) -> Tuple[ConfigurationMedium, Path]:
"""Find and return the directory in this tree that has remote-ing set up"""
possible_directory = working_dir
root = Path("/")
while possible_directory != root:
for meduim in CONFIG_MEDIUMS:
if meduim.is_workspace_root(possible_directory):
return meduim, possible_directory
possible_directory = possible_directory.parent
raise ConfigurationError(
f"Cannot resolve the remote workspace in {working_dir}")
def load_default_configuration_num(workspace_root: Path) -> int:
# If REMOTE_HOST_INDEX is set, that overrides settings in .remoteindex
env_index = os.environ.get("REMOTE_HOST_INDEX")
if env_index:
try:
return int(env_index)
except ValueError:
raise ConfigurationError(
f"REMOTE_HOST_INDEX env variable contains symbols other than numbers: '{env_index}'. "
"Please set the coorect index value to continue")
index_file = workspace_root / INDEX_FILE_NAME
if not index_file.exists():
return 0
# Configuration uses 1-base index and we need to have 0-based
text = index_file.read_text().strip()
try:
return int(text) - 1
except ValueError:
raise ConfigurationError(
f"File {index_file} contains symbols other than numbers: '{text}'. "
"Please remove it or replace the value to continue")
def _load_file(cls, path: Path):
if not path.exists():
return cls()
with path.open() as f:
try:
config = toml.load(f)
except ValueError as e:
raise ConfigurationError(
f"TOML file {path} is unparasble: {e}") from e
try:
return cls.parse_obj(config)
except ValidationError as e:
messages = []
for err in e.errors():
location = ".".join((str(x) for x in err["loc"]))
reason = err["msg"]
messages.append(f" - {location}: {reason}")
msg = "\n".join(messages)
raise ConfigurationError(
f"Invalid value in configuration file {path}:\n{msg}") from e
def load_config(self, workspace_root: Path) -> WorkspaceConfig:
local_config = load_local_config(workspace_root)
local_ignores_config = load_local_ignores_config(workspace_root)
# We might accidentally modify config value, so we need to create a copy of it
global_config = self.global_config.copy()
config_dict = {
field: _merge_field(field, global_config, local_config, local_ignores_config)
for field in WorkCycleConfig.__fields__
}
merged_config = WorkCycleConfig.parse_obj(config_dict)
if merged_config.hosts is None:
raise ConfigurationError("You need to provide at least one remote host to connect")
configurations = []
configuration_index = 0
for num, connection in enumerate(merged_config.hosts):
if connection.default:
configuration_index = num
configurations.append(
RemoteConfig(
host=connection.host,
directory=connection.directory or self._generate_remote_directory_from_path(workspace_root),
supports_gssapi=connection.supports_gssapi_auth,
label=connection.label,
port=connection.port,
)
)
ignores = SyncRules(
pull=_get_exclude(merged_config.pull, workspace_root),
push=_get_exclude(merged_config.push, workspace_root),
both=_get_exclude(merged_config.both, workspace_root) + [WORKSPACE_CONFIG],
)
includes = SyncRules(
pull=merged_config.pull.include if merged_config.pull else [],
push=merged_config.push.include if merged_config.push else [],
both=merged_config.both.include if merged_config.both else [],
)
return WorkspaceConfig(
root=workspace_root,
configurations=configurations,
default_configuration=configuration_index,
ignores=ignores,
includes=includes,
)
def load_config(self, workspace_root: Path) -> WorkspaceConfig:
configurations = load_configurations(workspace_root)
configuration_index = load_default_configuration_num(workspace_root)
if configuration_index > len(configurations) - 1:
raise ConfigurationError(
f"Configuration #{configuration_index + 1} requested but there are only {len(configurations)} declared"
)
ignores = load_ignores(workspace_root)
return WorkspaceConfig(
root=workspace_root,
configurations=configurations,
default_configuration=configuration_index,
ignores=ignores,
)
def parse_config_line(line: str) -> RemoteConfig:
# The line should look like this:
# sdas-ld2:.remotes/814f27f15f4e7a0842cada353dfc765a RSHELL=zsh
entry, *env_items = line.split(maxsplit=1)
shell, shell_options = _extract_shell_info(line, env_items)
parts = entry.split(":")
if len(parts) != 2:
raise ConfigurationError(
f"The configuration string is malformed: {parts}. Please use host-name:remote_dir format"
)
host, directory = parts
return RemoteConfig(host=host,
directory=Path(directory),
shell=shell,
shell_options=shell_options)
def load_local_config(workspace_root: Path) -> LocalConfig:
config_file = workspace_root / WORKSPACE_CONFIG
config = _load_file(LocalConfig, config_file)
if config.extends is not None:
duplicate_fields = []
for field in WorkCycleConfig.__fields__:
if getattr(config, field) and getattr(config.extends, field):
duplicate_fields.append(field)
if duplicate_fields:
fields_str = ",".join(duplicate_fields)
raise ConfigurationError(
f"Following fields are specified in for overwrite and extend in {config_file} file: {fields_str}."
)
return config