def _parse_and_validate(raw_config_list): """ Parse and validate settings. Merge settings from config files, global defaults, and command-line overrides. """ items = [] for raw in raw_config_list: # Validation. for key in CONFIGS_REQUIRED: if key not in raw or raw[key] is None: raise ConfigError("must specify '%s' in item config: %s" % (key, raw)) if "version_string" in raw and not _CONFIG_VERSION_RE.match(str(raw["version_string"])): raise ConfigError("invalid version string: '%s'" % raw["version_string"]) if "version_string" not in raw and "version_hashable" not in raw and "version_command" not in raw: raise ConfigError("must specify 'version_string', 'version_hashable', or 'version_command' in item config: %s" % raw) # Validate shell templates. # For these, we don't expand environment variables here, but instead do it at once at call time. for key in "upload_command", "download_command": try: strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"}) except ValueError as e: raise ConfigError("invalid command in config value for %s: %s" % (key, e)) # Normalize and expand environment variables. for key in "local_path", "remote_prefix", "remote_path": if key.startswith("/"): raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key) raw[key] = raw[key].rstrip("/") try: raw[key] = strif.expand_variables(raw[key], os.environ) except ValueError as e: raise ConfigError("invalid command in config value for %s: %s" % (key, e)) # Parse enums. try: raw["install_method"] = InstallMethod[raw["install_method"]] except KeyError: raise ConfigError("invalid install_method: %s" % raw["install_method"]) # Parse booleans. Values True and False may already be converted. try: if (type(raw["make_backup"]) is str): raw["make_backup"] = raw["make_backup"].lower() in ("on", "t", "true", "y", "yes") except KeyError: raise ConfigError("invalid make_backup: %s" % raw["make_backup"]) items.append(Config(**raw)) log.debug("final configs: %s", items) return items
def _parse_and_validate(raw_config_list): """ Parse and validate settings. Merge settings from config files, global defaults, and command-line overrides. """ items = [] for raw in raw_config_list: # Validation. for key in CONFIGS_REQUIRED: if key not in raw or raw[key] is None: raise ConfigError("must specify '%s' in item config: %s" % (key, raw)) if "version_string" in raw and not _CONFIG_VERSION_RE.match(str(raw["version_string"])): raise ConfigError("invalid version string: '%s'" % raw["version_string"]) if "version_string" not in raw and "version_hashable" not in raw and "version_command" not in raw: raise ConfigError("must specify 'version_string', 'version_hashable', or 'version_command' in item config: %s" % raw) # Validate shell templates. # For these, we don't expand environment variables here, but instead do it at once at call time. for key in "upload_command", "download_command": try: strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"}) except ValueError as e: raise ConfigError("invalid command in config value for %s: %s" % (key, e)) # Normalize and expand environment variables. for key in "local_path", "remote_prefix", "remote_path": if key.startswith("/"): raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key) raw[key] = raw[key].rstrip("/") try: raw[key] = strif.expand_variables(raw[key], os.environ) except ValueError as e: raise ConfigError("invalid command in config value for %s: %s" % (key, e)) # Parse enums. try: raw["install_method"] = InstallMethod[raw["install_method"]] except KeyError: raise ConfigError("invalid copy type: %s" % raw["install_method"]) # Parse booleans. try: raw["make_backup"] = raw["make_backup"] except KeyError: raise ConfigError("invalid copy type: %s" % raw["make_backup"]) items.append(Config(**raw)) log.debug("final configs: %s", items) return items
def _parse_validate(raw_config_list): for raw in raw_config_list: # Validation. for key in CONFIGS_REQUIRED: if key not in raw or raw[key] is None: raise ConfigError("must specify '%s' in item config: %s" % (key, raw)) if "version" in raw and not CONFIG_VERSION_RE.match(str( raw["version"])): raise ConfigError("invalid version string: '%s'" % raw["version"]) if "version" not in raw and "version_hashable" not in raw and "version_command" not in raw: raise ConfigError( "must specify 'version', 'version_hashable', or 'version_command' in item config: %s" % raw) # Validate shell templates. # For these, we don't expand environment variables here, but instead do it at once at call time. for key in "upload_command", "download_command": try: strif.shell_expand_to_popen(raw[key], { "REMOTE": "dummy", "LOCAL": "dummy" }) except ValueError as e: raise ConfigError( "invalid command in config value for %s: %s" % (key, e)) # Normalize and expand environment variables. for key in "local_path", "remote_prefix", "remote_path": if key.startswith("/"): raise ConfigError( "currently only support relative paths for local_path and remote_path: %s" % key) raw[key] = raw[key].rstrip("/") try: raw[key] = strif.expand_variables(raw[key], os.environ) except ValueError as e: raise ConfigError( "invalid command in config value for %s: %s" % (key, e)) # Parse enums. try: raw["copy_type"] = CopyType[raw["copy_type"]] except KeyError: raise ConfigError("invalid copy type: %s" % raw["copy_type"]) yield Config(**raw)
def _download_file(command_template, remote_loc, local_path): with atomic_output_file(local_path, make_parents=True) as temp_target: popenargs = shell_expand_to_popen(command_template, dict_merge(os.environ, {"REMOTE": remote_loc, "LOCAL": temp_target})) log.info("downloading: %s", " ".join(popenargs)) # TODO: Find a way to support force here. subprocess.check_call(popenargs, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
def unzip_dir(source_archive, target_dir): popenargs = shell_expand_to_popen(_autodetect_unzip_command(), { "ARCHIVE": source_archive, "DIR": target_dir }) cd_to = target_dir log.debug("using cwd: %s", cd_to) log.info("decompress: %s", " ".join(popenargs)) subprocess.check_call(popenargs, cwd=cd_to, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
def _upload_file(command_template, local_path, remote_loc): popenargs = shell_expand_to_popen( command_template, dict_merge(os.environ, { "REMOTE": remote_loc, "LOCAL": local_path })) log.info("uploading: %s", " ".join(popenargs)) # TODO: Find a way to support force here (e.g. add or remove -f to s4cmd) subprocess.check_call(popenargs, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
def _parse_validate(raw_config_list): for raw in raw_config_list: # Validation. for key in CONFIGS_REQUIRED: if key not in raw or raw[key] is None: raise ConfigError("must specify '%s' in item config: %s" % (key, raw)) if "version" in raw and not CONFIG_VERSION_RE.match(str(raw["version"])): raise ConfigError("invalid version string: '%s'" % raw["version"]) if "version" not in raw and "version_hashable" not in raw and "version_command" not in raw: raise ConfigError("must specify 'version', 'version_hashable', or 'version_command' in item config: %s" % raw) # Validate shell templates. # For these, we don't expand environment variables here, but instead do it at once at call time. for key in "upload_command", "download_command": try: strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"}) except ValueError as e: raise ConfigError("invalid command in config value for %s: %s" % (key, e)) # Normalize and expand environment variables. for key in "local_path", "remote_prefix", "remote_path": if key.startswith("/"): raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key) raw[key] = raw[key].rstrip("/") try: raw[key] = strif.expand_variables(raw[key], os.environ) except ValueError as e: raise ConfigError("invalid command in config value for %s: %s" % (key, e)) # Parse enums. try: raw["copy_type"] = CopyType[raw["copy_type"]] except KeyError: raise ConfigError("invalid copy type: %s" % raw["copy_type"]) yield Config(**raw)
def version_for(config): """ The version for an item is either the explicit version specified by the user, or the SHA1 hash of hashable file. """ bits = [] if config.version: bits.append(str(config.version)) if config.version_hashable: log.debug("computing sha1 of: %s", config.version_hashable) bits.append(file_sha1(config.version_hashable)) if config.version_command: log.debug("version command: %s", config.version_command) popenargs = shell_expand_to_popen(config.version_command, os.environ) output = subprocess.check_output(popenargs, stderr=SHELL_OUTPUT, stdin=DEV_NULL).strip() if not configs.CONFIG_VERSION_RE.match(output): raise configs.ConfigError("Invalid version output from version command: %r" % output) bits.append(output) return "-".join(bits)
def unzip_dir(source_archive, target_dir): popenargs = shell_expand_to_popen(_autodetect_unzip_command(), {"ARCHIVE": source_archive, "DIR": target_dir}) cd_to = target_dir log.debug("using cwd: %s", cd_to) log.info("decompress: %s", " ".join(popenargs)) subprocess.check_call(popenargs, cwd=cd_to, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
def _upload_file(command_template, local_path, remote_loc): popenargs = shell_expand_to_popen(command_template, dict_merge(os.environ, {"REMOTE": remote_loc, "LOCAL": local_path})) log.info("uploading: %s", " ".join(popenargs)) # TODO: Find a way to support force here (e.g. add or remove -f to s4cmd) subprocess.check_call(popenargs, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)