Ejemplo n.º 1
0
def _parse_and_validate(raw_config_list):
  """
  Parse and validate settings. Merge settings from config files, global defaults, and command-line overrides.
  """
  items = []
  for raw in raw_config_list:

    # Validation.
    for key in CONFIGS_REQUIRED:
      if key not in raw or raw[key] is None:
        raise ConfigError("must specify '%s' in item config: %s" % (key, raw))

    if "version_string" in raw and not _CONFIG_VERSION_RE.match(str(raw["version_string"])):
      raise ConfigError("invalid version string: '%s'" % raw["version_string"])
    if "version_string" not in raw and "version_hashable" not in raw and "version_command" not in raw:
      raise ConfigError("must specify 'version_string', 'version_hashable', or 'version_command' in item config: %s" % raw)

    # Validate shell templates.
    # For these, we don't expand environment variables here, but instead do it at once at call time.
    for key in "upload_command", "download_command":
      try:
        strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"})
      except ValueError as e:
        raise ConfigError("invalid command in config value for %s: %s" % (key, e))

    # Normalize and expand environment variables.
    for key in "local_path", "remote_prefix", "remote_path":
      if key.startswith("/"):
        raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key)
      raw[key] = raw[key].rstrip("/")

      try:
        raw[key] = strif.expand_variables(raw[key], os.environ)
      except ValueError as e:
        raise ConfigError("invalid command in config value for %s: %s" % (key, e))

    # Parse enums.
    try:
      raw["install_method"] = InstallMethod[raw["install_method"]]
    except KeyError:
      raise ConfigError("invalid install_method: %s" % raw["install_method"])

    # Parse booleans. Values True and False may already be converted.
    try:
      if (type(raw["make_backup"]) is str):
        raw["make_backup"] = raw["make_backup"].lower() in ("on", "t", "true", "y", "yes")
    except KeyError:
      raise ConfigError("invalid make_backup: %s" % raw["make_backup"])

    items.append(Config(**raw))

  log.debug("final configs: %s", items)
  return items
Ejemplo n.º 2
0
def _parse_and_validate(raw_config_list):
  """
  Parse and validate settings. Merge settings from config files, global defaults, and command-line overrides.
  """
  items = []
  for raw in raw_config_list:

    # Validation.
    for key in CONFIGS_REQUIRED:
      if key not in raw or raw[key] is None:
        raise ConfigError("must specify '%s' in item config: %s" % (key, raw))

    if "version_string" in raw and not _CONFIG_VERSION_RE.match(str(raw["version_string"])):
      raise ConfigError("invalid version string: '%s'" % raw["version_string"])
    if "version_string" not in raw and "version_hashable" not in raw and "version_command" not in raw:
      raise ConfigError("must specify 'version_string', 'version_hashable', or 'version_command' in item config: %s" % raw)

    # Validate shell templates.
    # For these, we don't expand environment variables here, but instead do it at once at call time.
    for key in "upload_command", "download_command":
      try:
        strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"})
      except ValueError as e:
        raise ConfigError("invalid command in config value for %s: %s" % (key, e))

    # Normalize and expand environment variables.
    for key in "local_path", "remote_prefix", "remote_path":
      if key.startswith("/"):
        raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key)
      raw[key] = raw[key].rstrip("/")

      try:
        raw[key] = strif.expand_variables(raw[key], os.environ)
      except ValueError as e:
        raise ConfigError("invalid command in config value for %s: %s" % (key, e))

    # Parse enums.
    try:
      raw["install_method"] = InstallMethod[raw["install_method"]]
    except KeyError:
      raise ConfigError("invalid copy type: %s" % raw["install_method"])

    # Parse booleans.
    try:
      raw["make_backup"] = raw["make_backup"]
    except KeyError:
      raise ConfigError("invalid copy type: %s" % raw["make_backup"])

    items.append(Config(**raw))

  log.debug("final configs: %s", items)
  return items
Ejemplo n.º 3
0
def _parse_validate(raw_config_list):
    for raw in raw_config_list:
        # Validation.
        for key in CONFIGS_REQUIRED:
            if key not in raw or raw[key] is None:
                raise ConfigError("must specify '%s' in item config: %s" %
                                  (key, raw))

        if "version" in raw and not CONFIG_VERSION_RE.match(str(
                raw["version"])):
            raise ConfigError("invalid version string: '%s'" % raw["version"])
        if "version" not in raw and "version_hashable" not in raw and "version_command" not in raw:
            raise ConfigError(
                "must specify 'version', 'version_hashable', or 'version_command' in item config: %s"
                % raw)

        # Validate shell templates.
        # For these, we don't expand environment variables here, but instead do it at once at call time.
        for key in "upload_command", "download_command":
            try:
                strif.shell_expand_to_popen(raw[key], {
                    "REMOTE": "dummy",
                    "LOCAL": "dummy"
                })
            except ValueError as e:
                raise ConfigError(
                    "invalid command in config value for %s: %s" % (key, e))

        # Normalize and expand environment variables.
        for key in "local_path", "remote_prefix", "remote_path":
            if key.startswith("/"):
                raise ConfigError(
                    "currently only support relative paths for local_path and remote_path: %s"
                    % key)
            raw[key] = raw[key].rstrip("/")

            try:
                raw[key] = strif.expand_variables(raw[key], os.environ)
            except ValueError as e:
                raise ConfigError(
                    "invalid command in config value for %s: %s" % (key, e))

        # Parse enums.
        try:
            raw["copy_type"] = CopyType[raw["copy_type"]]
        except KeyError:
            raise ConfigError("invalid copy type: %s" % raw["copy_type"])

        yield Config(**raw)
Ejemplo n.º 4
0
def _download_file(command_template, remote_loc, local_path):
  with atomic_output_file(local_path, make_parents=True) as temp_target:
    popenargs = shell_expand_to_popen(command_template,
                                      dict_merge(os.environ, {"REMOTE": remote_loc, "LOCAL": temp_target}))
    log.info("downloading: %s", " ".join(popenargs))
    # TODO: Find a way to support force here.
    subprocess.check_call(popenargs, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
Ejemplo n.º 5
0
def unzip_dir(source_archive, target_dir):
    popenargs = shell_expand_to_popen(_autodetect_unzip_command(), {
        "ARCHIVE": source_archive,
        "DIR": target_dir
    })
    cd_to = target_dir
    log.debug("using cwd: %s", cd_to)
    log.info("decompress: %s", " ".join(popenargs))
    subprocess.check_call(popenargs,
                          cwd=cd_to,
                          stdout=SHELL_OUTPUT,
                          stderr=SHELL_OUTPUT,
                          stdin=DEV_NULL)
Ejemplo n.º 6
0
def _upload_file(command_template, local_path, remote_loc):
    popenargs = shell_expand_to_popen(
        command_template,
        dict_merge(os.environ, {
            "REMOTE": remote_loc,
            "LOCAL": local_path
        }))
    log.info("uploading: %s", " ".join(popenargs))
    # TODO: Find a way to support force here (e.g. add or remove -f to s4cmd)
    subprocess.check_call(popenargs,
                          stdout=SHELL_OUTPUT,
                          stderr=SHELL_OUTPUT,
                          stdin=DEV_NULL)
Ejemplo n.º 7
0
def _parse_validate(raw_config_list):
  for raw in raw_config_list:
    # Validation.
    for key in CONFIGS_REQUIRED:
      if key not in raw or raw[key] is None:
        raise ConfigError("must specify '%s' in item config: %s" % (key, raw))

    if "version" in raw and not CONFIG_VERSION_RE.match(str(raw["version"])):
      raise ConfigError("invalid version string: '%s'" % raw["version"])
    if "version" not in raw and "version_hashable" not in raw and "version_command" not in raw:
      raise ConfigError("must specify 'version', 'version_hashable', or 'version_command' in item config: %s" % raw)

    # Validate shell templates.
    # For these, we don't expand environment variables here, but instead do it at once at call time.
    for key in "upload_command", "download_command":
      try:
        strif.shell_expand_to_popen(raw[key], {"REMOTE": "dummy", "LOCAL": "dummy"})
      except ValueError as e:
        raise ConfigError("invalid command in config value for %s: %s" % (key, e))

    # Normalize and expand environment variables.
    for key in "local_path", "remote_prefix", "remote_path":
      if key.startswith("/"):
        raise ConfigError("currently only support relative paths for local_path and remote_path: %s" % key)
      raw[key] = raw[key].rstrip("/")

      try:
        raw[key] = strif.expand_variables(raw[key], os.environ)
      except ValueError as e:
        raise ConfigError("invalid command in config value for %s: %s" % (key, e))

    # Parse enums.
    try:
      raw["copy_type"] = CopyType[raw["copy_type"]]
    except KeyError:
      raise ConfigError("invalid copy type: %s" % raw["copy_type"])

    yield Config(**raw)
Ejemplo n.º 8
0
def version_for(config):
  """
  The version for an item is either the explicit version specified by the user, or the SHA1 hash of hashable file.
  """
  bits = []
  if config.version:
    bits.append(str(config.version))
  if config.version_hashable:
    log.debug("computing sha1 of: %s", config.version_hashable)
    bits.append(file_sha1(config.version_hashable))
  if config.version_command:
    log.debug("version command: %s", config.version_command)
    popenargs = shell_expand_to_popen(config.version_command, os.environ)
    output = subprocess.check_output(popenargs, stderr=SHELL_OUTPUT, stdin=DEV_NULL).strip()
    if not configs.CONFIG_VERSION_RE.match(output):
      raise configs.ConfigError("Invalid version output from version command: %r" % output)
    bits.append(output)

  return "-".join(bits)
Ejemplo n.º 9
0
def unzip_dir(source_archive, target_dir):
  popenargs = shell_expand_to_popen(_autodetect_unzip_command(), {"ARCHIVE": source_archive, "DIR": target_dir})
  cd_to = target_dir
  log.debug("using cwd: %s", cd_to)
  log.info("decompress: %s", " ".join(popenargs))
  subprocess.check_call(popenargs, cwd=cd_to, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)
Ejemplo n.º 10
0
def _upload_file(command_template, local_path, remote_loc):
  popenargs = shell_expand_to_popen(command_template,
                                    dict_merge(os.environ, {"REMOTE": remote_loc, "LOCAL": local_path}))
  log.info("uploading: %s", " ".join(popenargs))
  # TODO: Find a way to support force here (e.g. add or remove -f to s4cmd)
  subprocess.check_call(popenargs, stdout=SHELL_OUTPUT, stderr=SHELL_OUTPUT, stdin=DEV_NULL)