Пример #1
0
  def wrapper_push(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path):
    # Create a simple rdiff-backup wrapper that will push

    template = textwrap.dedent("""\
        #!/bin/sh
        #
        # Push data to a PBS *-import instance.
        #

        LC_ALL=C
        export LC_ALL
        RDIFF_BACKUP=%(rdiffbackup_binary)s
        $RDIFF_BACKUP \\
                --remote-schema %(remote_schema)s \\
                --restore-as-of now \\
                --ignore-numerical-ids \\
                --force \\
                %(local_dir)s \\
                %(remote_dir)s
        """)

    template_dict = {
      'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
      'remote_schema': shlex.quote(remote_schema),
      'remote_dir': shlex.quote(remote_dir),
      'local_dir': shlex.quote(local_dir)
    }

    return self.createFile(
      name=rdiff_wrapper_path,
      content=template % template_dict,
      mode=0o700
    )
Пример #2
0
  def wrapper_push(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path):
    # Create a simple rdiff-backup wrapper that will push

    template = textwrap.dedent("""\
        #!/bin/sh
        #
        # Push data to a PBS *-import instance.
        #

        LC_ALL=C
        export LC_ALL
        RDIFF_BACKUP=%(rdiffbackup_binary)s
        $RDIFF_BACKUP \\
                --remote-schema %(remote_schema)s \\
                --restore-as-of now \\
                --ignore-numerical-ids \\
                --force \\
                %(local_dir)s \\
                %(remote_dir)s
        """)

    template_dict = {
      'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
      'remote_schema': shlex.quote(remote_schema),
      'remote_dir': shlex.quote(remote_dir),
      'local_dir': shlex.quote(local_dir)
    }

    return self.createFile(
      name=rdiff_wrapper_path,
      content=template % template_dict,
      mode=0o700
    )
Пример #3
0
  def createWrapper(self, name, command, parameters, comments=[],
        parameters_extra=False, environment=None):
    """
    Creates a very simple (one command) shell script for process replacement.
    Takes care of quoting.
    """

    lines = [ '#!/bin/sh' ]

    for comment in comments:
      lines.append('# %s' % comment)

    if environment:
      for key in environment:
        lines.append('export %s=%s' % (key, environment[key]))

    lines.append('exec %s' % shlex.quote(command))

    for param in parameters:
      if len(lines[-1]) < 40:
        lines[-1] += ' ' + shlex.quote(param)
      else:
        lines[-1] += ' \\'
        lines.append('\t' + shlex.quote(param))

    if parameters_extra:
        # pass-through further parameters
        lines[-1] += ' \\'
        lines.append('\t"$@"')

    content = '\n'.join(lines) + '\n'
    return self.createFile(name, content, 0700)
Пример #4
0
  def createWrapper(self, name, command, parameters, comments=[],
      parameters_extra=False, environment=None,
      pidfile=None
  ):
    """
    Creates a shell script for process replacement.
    Takes care of quoting.
    Takes care of #! line limitation when the wrapped command is a script.
    if pidfile parameter is specified, then it will make the wrapper a singleton,
    accepting to run only if no other instance is running.
    """

    lines = [ '#!/bin/sh' ]

    if comments:
      lines += '# ', '\n# '.join(comments), '\n'

    lines.append('COMMAND=' + shlex.quote(command))

    for key in environment or ():
      lines.append('export %s=%s' % (key, environment[key]))

    if pidfile:
      lines.append(dedent("""
          # Check for other instances
          pidfile=%s
          if [ -s $pidfile ]; then
            if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
              echo "Already running with pid $pid."
              exit 1
            fi
          fi
          echo $$ > $pidfile""" % shlex.quote(pidfile)))

    lines.append(dedent('''
    # If the wrapped command uses a shebang, execute the referenced
    # executable passing the script path as first argument.
    # This is to workaround the limitation of 127 characters in #!
    [ ! -f "$COMMAND" ] || {
      [ "`head -c2`" != "#!" ] || read -r EXE ARG
    } < "$COMMAND"

    exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))

    parameters = map(shlex.quote, parameters)
    if parameters_extra:
      # pass-through further parameters
      parameters.append('"$@"')
    for param in parameters:
      if len(lines[-1]) < 40:
        lines[-1] += ' ' + param
      else:
        lines[-1] += ' \\'
        lines.append('\t' + param)

    lines.append('')
    return self.createFile(name, '\n'.join(lines), 0700)
Пример #5
0
    def createWrapper(self,
                      name,
                      command,
                      parameters,
                      comments=[],
                      parameters_extra=False,
                      environment=None,
                      pidfile=None):
        """
    Creates a very simple (one command) shell script for process replacement.
    Takes care of quoting.
    if pidfile parameter is specified, then it will make the wrapper a singleton,
    accepting to run only if no other instance is running.
    """

        lines = ['#!/bin/sh']

        for comment in comments:
            lines.append('# %s' % comment)

        if environment:
            for key in environment:
                lines.append('export %s=%s' % (key, environment[key]))

        if pidfile:
            lines.append(
                dedent("""\
          # Check for other instances
          pidfile=%s
          if [ -e $pidfile ]; then
            pid=$(cat $pidfile)
            if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
              echo "Already running with pid $pid."
              exit 1
            else
              rm $pidfile
            fi
          fi
          echo $$ > $pidfile""" % (pidfile, command)))

        lines.append('exec %s' % shlex.quote(command))

        for param in parameters:
            if len(lines[-1]) < 40:
                lines[-1] += ' ' + shlex.quote(param)
            else:
                lines[-1] += ' \\'
                lines.append('\t' + shlex.quote(param))

        if parameters_extra:
            # pass-through further parameters
            lines[-1] += ' \\'
            lines.append('\t"$@"')

        content = '\n'.join(lines) + '\n'
        return self.createFile(name, content, 0700)
Пример #6
0
  def createWrapper(self, name, command, parameters, comments=[],
      parameters_extra=False, environment=None,
      pidfile=None
  ):
    """
    Creates a very simple (one command) shell script for process replacement.
    Takes care of quoting.
    if pidfile parameter is specified, then it will make the wrapper a singleton,
    accepting to run only if no other instance is running.
    """

    lines = [ '#!/bin/sh' ]

    for comment in comments:
      lines.append('# %s' % comment)

    if environment:
      for key in environment:
        lines.append('export %s=%s' % (key, environment[key]))

    if pidfile:
      lines.append(dedent("""\
          # Check for other instances
          pidfile=%s
          if [ -e $pidfile ]; then
            pid=$(cat $pidfile)
            if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
              echo "Already running with pid $pid."
              exit 1
            else
              rm $pidfile
            fi
          fi
          echo $$ > $pidfile""" % (pidfile, command)))

    lines.append('exec %s' % shlex.quote(command))

    for param in parameters:
      if len(lines[-1]) < 40:
        lines[-1] += ' ' + shlex.quote(param)
      else:
        lines[-1] += ' \\'
        lines.append('\t' + shlex.quote(param))

    if parameters_extra:
        # pass-through further parameters
        lines[-1] += ' \\'
        lines.append('\t"$@"')

    content = '\n'.join(lines) + '\n'
    return self.createFile(name, content, 0700)
Пример #7
0
    def createWrapper(self, path, args, env=None, **kw):
        """Create a wrapper script for process replacement"""
        assert args
        if kw:
            return self.createPythonScript(
                path, 'slapos.recipe.librecipe.execute.generic_exec',
                (args, env) if env else (args, ), kw)

        # Simple case: creates a basic shell script for process replacement.
        # This must be kept minimal to avoid code duplication with generic_exec.
        # In particular, do not implement workaround for shebang size limitation
        # here (note that this can't be done correctly with a POSIX shell, because
        # the process can't be given a name).

        lines = ['#!/bin/sh']

        if env:
            for k, v in sorted(six.iteritems(env)):
                lines.append('export %s=%s' % (k, shlex.quote(v)))

        lines.append('exec')

        args = list(map(shlex.quote, args))
        args.append('"$@"')
        for arg in args:
            if len(lines[-1]) < 40:
                lines[-1] += ' ' + arg
            else:
                lines[-1] += ' \\'
                lines.append('\t' + arg)

        lines.append('')
        return self.createFile(path, '\n'.join(lines), 0o700)
Пример #8
0
  def createWrapper(self, path, args, env=None, **kw):
    """Create a wrapper script for process replacement"""
    assert args
    if kw:
      return self.createPythonScript(path,
        'slapos.recipe.librecipe.execute.generic_exec',
        (args, env) if env else (args,), kw)

    # Simple case: creates a basic shell script for process replacement.
    # This must be kept minimal to avoid code duplication with generic_exec.
    # In particular, do not implement workaround for shebang size limitation
    # here (note that this can't be done correctly with a POSIX shell, because
    # the process can't be given a name).

    lines = ['#!/bin/sh']

    if env:
      for k, v in sorted(env.iteritems()):
        lines.append('export %s=%s' % (k, shlex.quote(v)))

    lines.append('exec')

    args = map(shlex.quote, args)
    args.append('"$@"')
    for arg in args:
      if len(lines[-1]) < 40:
        lines[-1] += ' ' + arg
      else:
        lines[-1] += ' \\'
        lines.append('\t' + arg)

    lines.append('')
    return self.createFile(path, '\n'.join(lines), 0700)
Пример #9
0
  def wrapper_pull(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path, remove_backup_older_than):
    # Wrap rdiff-backup call into a script that checks consistency of backup
    # We need to manually escape the remote schema

    template = textwrap.dedent("""\
        #!/bin/sh
        #
        # Pull data from a PBS *-export instance.
        #

        sigint()
        {
          exit 1
        }

        trap sigint INT  # we can CTRL-C for ease of debugging

        LC_ALL=C
        export LC_ALL
        is_first_backup=$(test -d %(rdiff_backup_data)s || echo yes)
        RDIFF_BACKUP=%(rdiffbackup_binary)s

        TMPDIR=%(tmpdir)s
        BACKUP_DIR=%(local_dir)s
        CORRUPTED_MSG="^Warning:\ Computed\ SHA1\ digest\ of\ "
        CANTFIND_MSG="^Warning:\ Cannot\ find\ SHA1\ digest\ for\ file\ "
        CORRUPTED_FILE=$TMPDIR/$$.rdiff_corrupted
        CANTFIND_FILE=$TMPDIR/$$.rdiff_cantfind

        SUCCEEDED=false

        # not using --fix-corrupted can lead to an infinite loop
        # in case of manual changes to the backup repository.

        CORRUPTED_ARGS=""
        if [ "$1" = "--fix-corrupted" ]; then
            VERIFY=$($RDIFF_BACKUP --verify $BACKUP_DIR 2>&1 >/dev/null)
            echo "$VERIFY" | egrep "$CORRUPTED_MSG" | sed "s/$CORRUPTED_MSG//g" > $CORRUPTED_FILE

            # Sometimes --verify reports this spurious warning:
            echo "$VERIFY" | egrep "$CANTFIND_MSG" | sed "s/$CANTFIND_MSG\(.*\),/--always-snapshot\ '\\1'/g" > $CANTFIND_FILE

            # There can be too many files, better not to provide them through separate command line parameters
            CORRUPTED_ARGS="--always-snapshot-fromfile $CORRUPTED_FILE --always-snapshot-fromfile $CANTFIND_FILE"

            if [ -s "$CORRUPTED_FILE" -o -s "$CANTFIND_FILE" ]; then
                echo Retransmitting $(cat "$CORRUPTED_FILE" "$CANTFIND_FILE" | wc -l) corrupted/missing files
            else
                echo "No corrupted or missing files to retransmit"
            fi
        fi

        $RDIFF_BACKUP \\
                $CORRUPTED_ARGS \\
                --remote-schema %(remote_schema)s \\
                %(remote_dir)s \\
                $BACKUP_DIR

        RDIFF_BACKUP_STATUS=$?

        [ "$CORRUPTED_ARGS" ] && rm -f "$CORRUPTED_FILE" "$CANTFIND_FILE"

        if [ ! $RDIFF_BACKUP_STATUS -eq 0 ]; then
            # Check the backup, go to the last consistent backup, so that next
            # run will be okay.
            echo "Checking backup directory..."
            $RDIFF_BACKUP --check-destination-dir $BACKUP_DIR
            if [ ! $? -eq 0 ]; then
                # Here, two possiblities:
                if [ is_first_backup ]; then
                    continue
                    # The first backup failed, and check-destination as well.
                    # we may want to remove the backup.
                else
                    continue
                    # The backup command has failed, while transferring an increment, and check-destination as well.
                    # XXX We may need to publish the failure and ask the the equeue, re-run this script again,
                    # instead do a push to the clone.
                fi
            fi
        else
            # Everything's okay, cleaning up...
            $RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force $BACKUP_DIR
        fi

        """)

    template_dict = {
      'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
      'rdiff_backup_data': shlex.quote(os.path.join(local_dir, 'rdiff-backup-data')),
      'remote_schema': shlex.quote(remote_schema),
      'remote_dir': shlex.quote(remote_dir),
      'local_dir': shlex.quote(local_dir),
      'tmpdir': '/tmp',
      'remove_backup_older_than': shlex.quote(remove_backup_older_than)
    }

    return self.createFile(
      name=rdiff_wrapper_path,
      content=template % template_dict,
      mode=0o700
    )
Пример #10
0
  def wrapper_pull(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path, remove_backup_older_than):
    # Wrap rdiff-backup call into a script that checks consistency of backup
    # We need to manually escape the remote schema

    template = textwrap.dedent("""\
        #!/bin/sh
        #
        # Pull data from a PBS *-export instance.
        #

        sigint()
        {
          exit 1
        }

        trap sigint INT  # we can CTRL-C for ease of debugging

        LC_ALL=C
        export LC_ALL
        is_first_backup=$(test -d %(rdiff_backup_data)s || echo yes)
        RDIFF_BACKUP=%(rdiffbackup_binary)s

        TMPDIR=%(tmpdir)s
        BACKUP_DIR=%(local_dir)s
        CORRUPTED_MSG="^Warning:\ Computed\ SHA1\ digest\ of\ "
        CANTFIND_MSG="^Warning:\ Cannot\ find\ SHA1\ digest\ for\ file\ "
        CORRUPTED_FILE=$TMPDIR/$$.rdiff_corrupted
        CANTFIND_FILE=$TMPDIR/$$.rdiff_cantfind

        SUCCEEDED=false

        # not using --fix-corrupted can lead to an infinite loop
        # in case of manual changes to the backup repository.

        CORRUPTED_ARGS=""
        if [ "$1" = "--fix-corrupted" ]; then
            VERIFY=$($RDIFF_BACKUP --verify $BACKUP_DIR 2>&1 >/dev/null)
            echo "$VERIFY" | egrep "$CORRUPTED_MSG" | sed "s/$CORRUPTED_MSG//g" > $CORRUPTED_FILE

            # Sometimes --verify reports this spurious warning:
            echo "$VERIFY" | egrep "$CANTFIND_MSG" | sed "s/$CANTFIND_MSG\(.*\),/--always-snapshot\ '\\1'/g" > $CANTFIND_FILE

            # There can be too many files, better not to provide them through separate command line parameters
            CORRUPTED_ARGS="--always-snapshot-fromfile $CORRUPTED_FILE --always-snapshot-fromfile $CANTFIND_FILE"

            if [ -s "$CORRUPTED_FILE" -o -s "$CANTFIND_FILE" ]; then
                echo Retransmitting $(cat "$CORRUPTED_FILE" "$CANTFIND_FILE" | wc -l) corrupted/missing files
            else
                echo "No corrupted or missing files to retransmit"
            fi
        fi

        $RDIFF_BACKUP \\
                $CORRUPTED_ARGS \\
                --remote-schema %(remote_schema)s \\
                %(remote_dir)s \\
                $BACKUP_DIR

        RDIFF_BACKUP_STATUS=$?

        [ "$CORRUPTED_ARGS" ] && rm -f "$CORRUPTED_FILE" "$CANTFIND_FILE"

        if [ ! $RDIFF_BACKUP_STATUS -eq 0 ]; then
            # Check the backup, go to the last consistent backup, so that next
            # run will be okay.
            echo "Checking backup directory..."
            $RDIFF_BACKUP --check-destination-dir $BACKUP_DIR
            if [ ! $? -eq 0 ]; then
                # Here, two possiblities:
                if [ is_first_backup ]; then
                    continue
                    # The first backup failed, and check-destination as well.
                    # we may want to remove the backup.
                else
                    continue
                    # The backup command has failed, while transferring an increment, and check-destination as well.
                    # XXX We may need to publish the failure and ask the the equeue, re-run this script again,
                    # instead do a push to the clone.
                fi
            fi
        else
            # Everything's okay, cleaning up...
            $RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force $BACKUP_DIR
        fi

        """)

    template_dict = {
      'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
      'rdiff_backup_data': shlex.quote(os.path.join(local_dir, 'rdiff-backup-data')),
      'remote_schema': shlex.quote(remote_schema),
      'remote_dir': shlex.quote(remote_dir),
      'local_dir': shlex.quote(local_dir),
      'tmpdir': '/tmp',
      'remove_backup_older_than': shlex.quote(remove_backup_older_than)
    }

    return self.createFile(
      name=rdiff_wrapper_path,
      content=template % template_dict,
      mode=0o700
    )