def _parse_and_validate(raw_config_list):
"""
Parse and validate settings. Merge settings from config files, global defaults, and command-line overrides.
"""
items = []
for raw in raw_config_list:
# Validation.
for key in CONFIGS_REQUIRED:
if key not in raw or raw[key] is None:
raise ConfigError("must specify '%s' in item config: %s" % (key, raw))
if "version_string" in raw and not _CONFIG_VERSION_RE.match(str(raw["version_string"])):
raise ConfigError("invalid version string: '%s'" % raw["version_string"])
if "version_string" not in raw and "version_hashable" not in raw and "version_command" not in raw:
raise ConfigError("must specify 'version_string', 'version_hashable', or 'version_command' in item config: %s" % raw)
# Validate shell templates.
# For these, we don't expand environment variables here, but instead do it at once at call time.
for key in "upload_command", "download_command":
try:
strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"})
except ValueError as e:
raise ConfigError("invalid command in config value for %s: %s" % (key, e))
# Normalize and expand environment variables.
for key in "local_path", "remote_prefix", "remote_path":
if key.startswith("/"):
raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key)
raw[key] = raw[key].rstrip("/")
try:
raw[key] = strif.expand_variables(raw[key], os.environ)
except ValueError as e:
raise ConfigError("invalid command in config value for %s: %s" % (key, e))
# Parse enums.
try:
raw["install_method"] = InstallMethod[raw["install_method"]]
except KeyError:
raise ConfigError("invalid install_method: %s" % raw["install_method"])
# Parse booleans. Values True and False may already be converted.
try:
if (type(raw["make_backup"]) is str):
raw["make_backup"] = raw["make_backup"].lower() in ("on", "t", "true", "y", "yes")
except KeyError:
raise ConfigError("invalid make_backup: %s" % raw["make_backup"])
items.append(Config(**raw))
log.debug("final configs: %s", items)
return items
def _parse_and_validate(raw_config_list):
"""
Parse and validate settings. Merge settings from config files, global defaults, and command-line overrides.
"""
items = []
for raw in raw_config_list:
# Validation.
for key in CONFIGS_REQUIRED:
if key not in raw or raw[key] is None:
raise ConfigError("must specify '%s' in item config: %s" % (key, raw))
if "version_string" in raw and not _CONFIG_VERSION_RE.match(str(raw["version_string"])):
raise ConfigError("invalid version string: '%s'" % raw["version_string"])
if "version_string" not in raw and "version_hashable" not in raw and "version_command" not in raw:
raise ConfigError("must specify 'version_string', 'version_hashable', or 'version_command' in item config: %s" % raw)
# Validate shell templates.
# For these, we don't expand environment variables here, but instead do it at once at call time.
for key in "upload_command", "download_command":
try:
strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"})
except ValueError as e:
raise ConfigError("invalid command in config value for %s: %s" % (key, e))
# Normalize and expand environment variables.
for key in "local_path", "remote_prefix", "remote_path":
if key.startswith("/"):
raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key)
raw[key] = raw[key].rstrip("/")
try:
raw[key] = strif.expand_variables(raw[key], os.environ)
except ValueError as e:
raise ConfigError("invalid command in config value for %s: %s" % (key, e))
# Parse enums.
try:
raw["install_method"] = InstallMethod[raw["install_method"]]
except KeyError:
raise ConfigError("invalid copy type: %s" % raw["install_method"])
# Parse booleans.
try:
raw["make_backup"] = raw["make_backup"]
except KeyError:
raise ConfigError("invalid copy type: %s" % raw["make_backup"])
items.append(Config(**raw))
log.debug("final configs: %s", items)
return items
def _parse_validate(raw_config_list):
for raw in raw_config_list:
# Validation.
for key in CONFIGS_REQUIRED:
if key not in raw or raw[key] is None:
raise ConfigError("must specify '%s' in item config: %s" %
(key, raw))
if "version" in raw and not CONFIG_VERSION_RE.match(str(
raw["version"])):
raise ConfigError("invalid version string: '%s'" % raw["version"])
if "version" not in raw and "version_hashable" not in raw and "version_command" not in raw:
raise ConfigError(
"must specify 'version', 'version_hashable', or 'version_command' in item config: %s"
% raw)
# Validate shell templates.
# For these, we don't expand environment variables here, but instead do it at once at call time.
for key in "upload_command", "download_command":
try:
strif.shell_expand_to_popen(raw[key], {
"REMOTE": "dummy",
"LOCAL": "dummy"
})
except ValueError as e:
raise ConfigError(
"invalid command in config value for %s: %s" % (key, e))
# Normalize and expand environment variables.
for key in "local_path", "remote_prefix", "remote_path":
if key.startswith("/"):
raise ConfigError(
"currently only support relative paths for local_path and remote_path: %s"
% key)
raw[key] = raw[key].rstrip("/")
try:
raw[key] = strif.expand_variables(raw[key], os.environ)
except ValueError as e:
raise ConfigError(
"invalid command in config value for %s: %s" % (key, e))
# Parse enums.
try:
raw["copy_type"] = CopyType[raw["copy_type"]]
except KeyError:
raise ConfigError("invalid copy type: %s" % raw["copy_type"])
yield Config(**raw)
def _download_file(command_template, remote_loc, local_path):
with atomic_output_file(local_path, make_parents=True) as temp_target:
popenargs = shell_expand_to_popen(command_template,
dict_merge(os.environ, {"REMOTE": remote_loc, "LOCAL": temp_target}))
log.info("downloading: %s", " ".join(popenargs))
# TODO: Find a way to support force here.
subprocess.check_call(popenargs, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
def unzip_dir(source_archive, target_dir):
popenargs = shell_expand_to_popen(_autodetect_unzip_command(), {
"ARCHIVE": source_archive,
"DIR": target_dir
})
cd_to = target_dir
log.debug("using cwd: %s", cd_to)
log.info("decompress: %s", " ".join(popenargs))
subprocess.check_call(popenargs,
cwd=cd_to,
stdout=SHELL_OUTPUT,
stderr=SHELL_OUTPUT,
stdin=DEV_NULL)
def _upload_file(command_template, local_path, remote_loc):
popenargs = shell_expand_to_popen(
command_template,
dict_merge(os.environ, {
"REMOTE": remote_loc,
"LOCAL": local_path
}))
log.info("uploading: %s", " ".join(popenargs))
# TODO: Find a way to support force here (e.g. add or remove -f to s4cmd)
subprocess.check_call(popenargs,
stdout=SHELL_OUTPUT,
stderr=SHELL_OUTPUT,
stdin=DEV_NULL)
def _parse_validate(raw_config_list):
for raw in raw_config_list:
# Validation.
for key in CONFIGS_REQUIRED:
if key not in raw or raw[key] is None:
raise ConfigError("must specify '%s' in item config: %s" % (key, raw))
if "version" in raw and not CONFIG_VERSION_RE.match(str(raw["version"])):
raise ConfigError("invalid version string: '%s'" % raw["version"])
if "version" not in raw and "version_hashable" not in raw and "version_command" not in raw:
raise ConfigError("must specify 'version', 'version_hashable', or 'version_command' in item config: %s" % raw)
# Validate shell templates.
# For these, we don't expand environment variables here, but instead do it at once at call time.
for key in "upload_command", "download_command":
try:
strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"})
except ValueError as e:
raise ConfigError("invalid command in config value for %s: %s" % (key, e))
# Normalize and expand environment variables.
for key in "local_path", "remote_prefix", "remote_path":
if key.startswith("/"):
raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key)
raw[key] = raw[key].rstrip("/")
try:
raw[key] = strif.expand_variables(raw[key], os.environ)
except ValueError as e:
raise ConfigError("invalid command in config value for %s: %s" % (key, e))
# Parse enums.
try:
raw["copy_type"] = CopyType[raw["copy_type"]]
except KeyError:
raise ConfigError("invalid copy type: %s" % raw["copy_type"])
yield Config(**raw)
def version_for(config):
"""
The version for an item is either the explicit version specified by the user, or the SHA1 hash of hashable file.
"""
bits = []
if config.version:
bits.append(str(config.version))
if config.version_hashable:
log.debug("computing sha1 of: %s", config.version_hashable)
bits.append(file_sha1(config.version_hashable))
if config.version_command:
log.debug("version command: %s", config.version_command)
popenargs = shell_expand_to_popen(config.version_command, os.environ)
output = subprocess.check_output(popenargs, stderr=SHELL_OUTPUT, stdin=DEV_NULL).strip()
if not configs.CONFIG_VERSION_RE.match(output):
raise configs.ConfigError("Invalid version output from version command: %r" % output)
bits.append(output)
return "-".join(bits)
def unzip_dir(source_archive, target_dir):
popenargs = shell_expand_to_popen(_autodetect_unzip_command(), {"ARCHIVE": source_archive, "DIR": target_dir})
cd_to = target_dir
log.debug("using cwd: %s", cd_to)
log.info("decompress: %s", " ".join(popenargs))
subprocess.check_call(popenargs, cwd=cd_to, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
def _upload_file(command_template, local_path, remote_loc):
popenargs = shell_expand_to_popen(command_template,
dict_merge(os.environ, {"REMOTE": remote_loc, "LOCAL": local_path}))
log.info("uploading: %s", " ".join(popenargs))
# TODO: Find a way to support force here (e.g. add or remove -f to s4cmd)
subprocess.check_call(popenargs, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)