def wrapper_push(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path):
# Create a simple rdiff-backup wrapper that will push
template = textwrap.dedent("""\
#!/bin/sh
#
# Push data to a PBS *-import instance.
#
LC_ALL=C
export LC_ALL
RDIFF_BACKUP=%(rdiffbackup_binary)s
$RDIFF_BACKUP \\
--remote-schema %(remote_schema)s \\
--restore-as-of now \\
--ignore-numerical-ids \\
--force \\
%(local_dir)s \\
%(remote_dir)s
""")
template_dict = {
'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
'remote_schema': shlex.quote(remote_schema),
'remote_dir': shlex.quote(remote_dir),
'local_dir': shlex.quote(local_dir)
}
return self.createFile(
name=rdiff_wrapper_path,
content=template % template_dict,
mode=0o700
)
def wrapper_push(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path):
# Create a simple rdiff-backup wrapper that will push
template = textwrap.dedent("""\
#!/bin/sh
#
# Push data to a PBS *-import instance.
#
LC_ALL=C
export LC_ALL
RDIFF_BACKUP=%(rdiffbackup_binary)s
$RDIFF_BACKUP \\
--remote-schema %(remote_schema)s \\
--restore-as-of now \\
--ignore-numerical-ids \\
--force \\
%(local_dir)s \\
%(remote_dir)s
""")
template_dict = {
'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
'remote_schema': shlex.quote(remote_schema),
'remote_dir': shlex.quote(remote_dir),
'local_dir': shlex.quote(local_dir)
}
return self.createFile(
name=rdiff_wrapper_path,
content=template % template_dict,
mode=0o700
)
def createWrapper(self, name, command, parameters, comments=[],
parameters_extra=False, environment=None):
"""
Creates a very simple (one command) shell script for process replacement.
Takes care of quoting.
"""
lines = [ '#!/bin/sh' ]
for comment in comments:
lines.append('# %s' % comment)
if environment:
for key in environment:
lines.append('export %s=%s' % (key, environment[key]))
lines.append('exec %s' % shlex.quote(command))
for param in parameters:
if len(lines[-1]) < 40:
lines[-1] += ' ' + shlex.quote(param)
else:
lines[-1] += ' \\'
lines.append('\t' + shlex.quote(param))
if parameters_extra:
# pass-through further parameters
lines[-1] += ' \\'
lines.append('\t"$@"')
content = '\n'.join(lines) + '\n'
return self.createFile(name, content, 0700)
def createWrapper(self, name, command, parameters, comments=[],
parameters_extra=False, environment=None,
pidfile=None
):
"""
Creates a shell script for process replacement.
Takes care of quoting.
Takes care of #! line limitation when the wrapped command is a script.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
"""
lines = [ '#!/bin/sh' ]
if comments:
lines += '# ', '\n# '.join(comments), '\n'
lines.append('COMMAND=' + shlex.quote(command))
for key in environment or ():
lines.append('export %s=%s' % (key, environment[key]))
if pidfile:
lines.append(dedent("""
# Check for other instances
pidfile=%s
if [ -s $pidfile ]; then
if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
echo "Already running with pid $pid."
exit 1
fi
fi
echo $$ > $pidfile""" % shlex.quote(pidfile)))
lines.append(dedent('''
# If the wrapped command uses a shebang, execute the referenced
# executable passing the script path as first argument.
# This is to workaround the limitation of 127 characters in #!
[ ! -f "$COMMAND" ] || {
[ "`head -c2`" != "#!" ] || read -r EXE ARG
} < "$COMMAND"
exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))
parameters = map(shlex.quote, parameters)
if parameters_extra:
# pass-through further parameters
parameters.append('"$@"')
for param in parameters:
if len(lines[-1]) < 40:
lines[-1] += ' ' + param
else:
lines[-1] += ' \\'
lines.append('\t' + param)
lines.append('')
return self.createFile(name, '\n'.join(lines), 0700)
def createWrapper(self,
name,
command,
parameters,
comments=[],
parameters_extra=False,
environment=None,
pidfile=None):
"""
Creates a very simple (one command) shell script for process replacement.
Takes care of quoting.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
"""
lines = ['#!/bin/sh']
for comment in comments:
lines.append('# %s' % comment)
if environment:
for key in environment:
lines.append('export %s=%s' % (key, environment[key]))
if pidfile:
lines.append(
dedent("""\
# Check for other instances
pidfile=%s
if [ -e $pidfile ]; then
pid=$(cat $pidfile)
if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
echo "Already running with pid $pid."
exit 1
else
rm $pidfile
fi
fi
echo $$ > $pidfile""" % (pidfile, command)))
lines.append('exec %s' % shlex.quote(command))
for param in parameters:
if len(lines[-1]) < 40:
lines[-1] += ' ' + shlex.quote(param)
else:
lines[-1] += ' \\'
lines.append('\t' + shlex.quote(param))
if parameters_extra:
# pass-through further parameters
lines[-1] += ' \\'
lines.append('\t"$@"')
content = '\n'.join(lines) + '\n'
return self.createFile(name, content, 0700)
def createWrapper(self, name, command, parameters, comments=[],
parameters_extra=False, environment=None,
pidfile=None
):
"""
Creates a very simple (one command) shell script for process replacement.
Takes care of quoting.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
"""
lines = [ '#!/bin/sh' ]
for comment in comments:
lines.append('# %s' % comment)
if environment:
for key in environment:
lines.append('export %s=%s' % (key, environment[key]))
if pidfile:
lines.append(dedent("""\
# Check for other instances
pidfile=%s
if [ -e $pidfile ]; then
pid=$(cat $pidfile)
if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
echo "Already running with pid $pid."
exit 1
else
rm $pidfile
fi
fi
echo $$ > $pidfile""" % (pidfile, command)))
lines.append('exec %s' % shlex.quote(command))
for param in parameters:
if len(lines[-1]) < 40:
lines[-1] += ' ' + shlex.quote(param)
else:
lines[-1] += ' \\'
lines.append('\t' + shlex.quote(param))
if parameters_extra:
# pass-through further parameters
lines[-1] += ' \\'
lines.append('\t"$@"')
content = '\n'.join(lines) + '\n'
return self.createFile(name, content, 0700)
def createWrapper(self, path, args, env=None, **kw):
"""Create a wrapper script for process replacement"""
assert args
if kw:
return self.createPythonScript(
path, 'slapos.recipe.librecipe.execute.generic_exec',
(args, env) if env else (args, ), kw)
# Simple case: creates a basic shell script for process replacement.
# This must be kept minimal to avoid code duplication with generic_exec.
# In particular, do not implement workaround for shebang size limitation
# here (note that this can't be done correctly with a POSIX shell, because
# the process can't be given a name).
lines = ['#!/bin/sh']
if env:
for k, v in sorted(six.iteritems(env)):
lines.append('export %s=%s' % (k, shlex.quote(v)))
lines.append('exec')
args = list(map(shlex.quote, args))
args.append('"$@"')
for arg in args:
if len(lines[-1]) < 40:
lines[-1] += ' ' + arg
else:
lines[-1] += ' \\'
lines.append('\t' + arg)
lines.append('')
return self.createFile(path, '\n'.join(lines), 0o700)
def createWrapper(self, path, args, env=None, **kw):
"""Create a wrapper script for process replacement"""
assert args
if kw:
return self.createPythonScript(path,
'slapos.recipe.librecipe.execute.generic_exec',
(args, env) if env else (args,), kw)
# Simple case: creates a basic shell script for process replacement.
# This must be kept minimal to avoid code duplication with generic_exec.
# In particular, do not implement workaround for shebang size limitation
# here (note that this can't be done correctly with a POSIX shell, because
# the process can't be given a name).
lines = ['#!/bin/sh']
if env:
for k, v in sorted(env.iteritems()):
lines.append('export %s=%s' % (k, shlex.quote(v)))
lines.append('exec')
args = map(shlex.quote, args)
args.append('"$@"')
for arg in args:
if len(lines[-1]) < 40:
lines[-1] += ' ' + arg
else:
lines[-1] += ' \\'
lines.append('\t' + arg)
lines.append('')
return self.createFile(path, '\n'.join(lines), 0700)
def wrapper_pull(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path, remove_backup_older_than):
# Wrap rdiff-backup call into a script that checks consistency of backup
# We need to manually escape the remote schema
template = textwrap.dedent("""\
#!/bin/sh
#
# Pull data from a PBS *-export instance.
#
sigint()
{
exit 1
}
trap sigint INT # we can CTRL-C for ease of debugging
LC_ALL=C
export LC_ALL
is_first_backup=$(test -d %(rdiff_backup_data)s || echo yes)
RDIFF_BACKUP=%(rdiffbackup_binary)s
TMPDIR=%(tmpdir)s
BACKUP_DIR=%(local_dir)s
CORRUPTED_MSG="^Warning:\ Computed\ SHA1\ digest\ of\ "
CANTFIND_MSG="^Warning:\ Cannot\ find\ SHA1\ digest\ for\ file\ "
CORRUPTED_FILE=$TMPDIR/$$.rdiff_corrupted
CANTFIND_FILE=$TMPDIR/$$.rdiff_cantfind
SUCCEEDED=false
# not using --fix-corrupted can lead to an infinite loop
# in case of manual changes to the backup repository.
CORRUPTED_ARGS=""
if [ "$1" = "--fix-corrupted" ]; then
VERIFY=$($RDIFF_BACKUP --verify $BACKUP_DIR 2>&1 >/dev/null)
echo "$VERIFY" | egrep "$CORRUPTED_MSG" | sed "s/$CORRUPTED_MSG//g" > $CORRUPTED_FILE
# Sometimes --verify reports this spurious warning:
echo "$VERIFY" | egrep "$CANTFIND_MSG" | sed "s/$CANTFIND_MSG\(.*\),/--always-snapshot\ '\\1'/g" > $CANTFIND_FILE
# There can be too many files, better not to provide them through separate command line parameters
CORRUPTED_ARGS="--always-snapshot-fromfile $CORRUPTED_FILE --always-snapshot-fromfile $CANTFIND_FILE"
if [ -s "$CORRUPTED_FILE" -o -s "$CANTFIND_FILE" ]; then
echo Retransmitting $(cat "$CORRUPTED_FILE" "$CANTFIND_FILE" | wc -l) corrupted/missing files
else
echo "No corrupted or missing files to retransmit"
fi
fi
$RDIFF_BACKUP \\
$CORRUPTED_ARGS \\
--remote-schema %(remote_schema)s \\
%(remote_dir)s \\
$BACKUP_DIR
RDIFF_BACKUP_STATUS=$?
[ "$CORRUPTED_ARGS" ] && rm -f "$CORRUPTED_FILE" "$CANTFIND_FILE"
if [ ! $RDIFF_BACKUP_STATUS -eq 0 ]; then
# Check the backup, go to the last consistent backup, so that next
# run will be okay.
echo "Checking backup directory..."
$RDIFF_BACKUP --check-destination-dir $BACKUP_DIR
if [ ! $? -eq 0 ]; then
# Here, two possiblities:
if [ is_first_backup ]; then
continue
# The first backup failed, and check-destination as well.
# we may want to remove the backup.
else
continue
# The backup command has failed, while transferring an increment, and check-destination as well.
# XXX We may need to publish the failure and ask the the equeue, re-run this script again,
# instead do a push to the clone.
fi
fi
else
# Everything's okay, cleaning up...
$RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force $BACKUP_DIR
fi
""")
template_dict = {
'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
'rdiff_backup_data': shlex.quote(os.path.join(local_dir, 'rdiff-backup-data')),
'remote_schema': shlex.quote(remote_schema),
'remote_dir': shlex.quote(remote_dir),
'local_dir': shlex.quote(local_dir),
'tmpdir': '/tmp',
'remove_backup_older_than': shlex.quote(remove_backup_older_than)
}
return self.createFile(
name=rdiff_wrapper_path,
content=template % template_dict,
mode=0o700
)
def wrapper_pull(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path, remove_backup_older_than):
# Wrap rdiff-backup call into a script that checks consistency of backup
# We need to manually escape the remote schema
template = textwrap.dedent("""\
#!/bin/sh
#
# Pull data from a PBS *-export instance.
#
sigint()
{
exit 1
}
trap sigint INT # we can CTRL-C for ease of debugging
LC_ALL=C
export LC_ALL
is_first_backup=$(test -d %(rdiff_backup_data)s || echo yes)
RDIFF_BACKUP=%(rdiffbackup_binary)s
TMPDIR=%(tmpdir)s
BACKUP_DIR=%(local_dir)s
CORRUPTED_MSG="^Warning:\ Computed\ SHA1\ digest\ of\ "
CANTFIND_MSG="^Warning:\ Cannot\ find\ SHA1\ digest\ for\ file\ "
CORRUPTED_FILE=$TMPDIR/$$.rdiff_corrupted
CANTFIND_FILE=$TMPDIR/$$.rdiff_cantfind
SUCCEEDED=false
# not using --fix-corrupted can lead to an infinite loop
# in case of manual changes to the backup repository.
CORRUPTED_ARGS=""
if [ "$1" = "--fix-corrupted" ]; then
VERIFY=$($RDIFF_BACKUP --verify $BACKUP_DIR 2>&1 >/dev/null)
echo "$VERIFY" | egrep "$CORRUPTED_MSG" | sed "s/$CORRUPTED_MSG//g" > $CORRUPTED_FILE
# Sometimes --verify reports this spurious warning:
echo "$VERIFY" | egrep "$CANTFIND_MSG" | sed "s/$CANTFIND_MSG\(.*\),/--always-snapshot\ '\\1'/g" > $CANTFIND_FILE
# There can be too many files, better not to provide them through separate command line parameters
CORRUPTED_ARGS="--always-snapshot-fromfile $CORRUPTED_FILE --always-snapshot-fromfile $CANTFIND_FILE"
if [ -s "$CORRUPTED_FILE" -o -s "$CANTFIND_FILE" ]; then
echo Retransmitting $(cat "$CORRUPTED_FILE" "$CANTFIND_FILE" | wc -l) corrupted/missing files
else
echo "No corrupted or missing files to retransmit"
fi
fi
$RDIFF_BACKUP \\
$CORRUPTED_ARGS \\
--remote-schema %(remote_schema)s \\
%(remote_dir)s \\
$BACKUP_DIR
RDIFF_BACKUP_STATUS=$?
[ "$CORRUPTED_ARGS" ] && rm -f "$CORRUPTED_FILE" "$CANTFIND_FILE"
if [ ! $RDIFF_BACKUP_STATUS -eq 0 ]; then
# Check the backup, go to the last consistent backup, so that next
# run will be okay.
echo "Checking backup directory..."
$RDIFF_BACKUP --check-destination-dir $BACKUP_DIR
if [ ! $? -eq 0 ]; then
# Here, two possiblities:
if [ is_first_backup ]; then
continue
# The first backup failed, and check-destination as well.
# we may want to remove the backup.
else
continue
# The backup command has failed, while transferring an increment, and check-destination as well.
# XXX We may need to publish the failure and ask the the equeue, re-run this script again,
# instead do a push to the clone.
fi
fi
else
# Everything's okay, cleaning up...
$RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force $BACKUP_DIR
fi
""")
template_dict = {
'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
'rdiff_backup_data': shlex.quote(os.path.join(local_dir, 'rdiff-backup-data')),
'remote_schema': shlex.quote(remote_schema),
'remote_dir': shlex.quote(remote_dir),
'local_dir': shlex.quote(local_dir),
'tmpdir': '/tmp',
'remove_backup_older_than': shlex.quote(remove_backup_older_than)
}
return self.createFile(
name=rdiff_wrapper_path,
content=template % template_dict,
mode=0o700
)
