def test_splitCommands(self):
ret = list(
tools.splitCommands(['echo foo;'],
head='echo start;',
tail='echo end',
maxLength=40))
self.assertEqual(len(ret), 1)
self.assertEqual(ret[0], 'echo start;echo foo;echo end')
ret = list(
tools.splitCommands(['echo foo;'] * 3,
head='echo start;',
tail='echo end',
maxLength=40))
self.assertEqual(len(ret), 2)
self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo end')
self.assertEqual(ret[1], 'echo start;echo foo;echo end')
ret = list(
tools.splitCommands(['echo foo;'] * 3,
head='echo start;',
tail='echo end',
maxLength=0))
self.assertEqual(len(ret), 1)
self.assertEqual(ret[0],
'echo start;echo foo;echo foo;echo foo;echo end')
ret = list(
tools.splitCommands(['echo foo;'] * 3,
head='echo start;',
tail='echo end',
maxLength=-10))
self.assertEqual(len(ret), 1)
self.assertEqual(ret[0],
'echo start;echo foo;echo foo;echo foo;echo end')
def test_splitCommands(self):
ret = list(tools.splitCommands(['echo foo;'],
head = 'echo start;',
tail = 'echo end',
maxLength = 40))
self.assertEqual(len(ret), 1)
self.assertEqual(ret[0], 'echo start;echo foo;echo end')
ret = list(tools.splitCommands(['echo foo;']*3,
head = 'echo start;',
tail = 'echo end',
maxLength = 40))
self.assertEqual(len(ret), 2)
self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo end')
self.assertEqual(ret[1], 'echo start;echo foo;echo end')
ret = list(tools.splitCommands(['echo foo;']*3,
head = 'echo start;',
tail = 'echo end',
maxLength = 0))
self.assertEqual(len(ret), 1)
self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo foo;echo end')
ret = list(tools.splitCommands(['echo foo;']*3,
head = 'echo start;',
tail = 'echo end',
maxLength = -10))
self.assertEqual(len(ret), 1)
self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo foo;echo end')
def check_remote_commands(self, retry = False):
"""
try all relevant commands for take_snapshot on remote host.
specialy embedded Linux devices using 'BusyBox' sometimes doesn't
support everything that is need to run backintime.
also check for hardlink-support on remote host.
"""
logger.debug('Check remote commands', self)
def maxArg():
if retry:
raise MountException("Checking commands on remote host didn't return any output. "
"We already checked the maximum argument lenght but it seem like "
"there is an other problem")
logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
self)
import sshMaxArg
mid = sshMaxArg.test_ssh_max_arg(self.user_host)
sshMaxArg.reportResult(self.host, mid)
self.config.set_ssh_max_arg_length(mid, self.profile_id)
return self.check_remote_commands(retry = True)
#check rsync
tmp_file = tempfile.mkstemp()[1]
rsync = tools.get_rsync_prefix( self.config ) + ' --dry-run --chmod=Du+wx %s ' % tmp_file
rsync += '"%s@%s:%s"' % (self.user, self.host, self.path)
logger.debug('Check rsync command: %s' %rsync, self)
#use os.system for compatiblity with snapshots.py
err = os.system(rsync)
if err:
logger.debug('Rsync command returnd error: %s' %err, self)
os.remove(tmp_file)
raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'command' : rsync, 'err' : err})
os.remove(tmp_file)
#check cp chmod find and rm
remote_tmp_dir = os.path.join(self.path, 'tmp_%s' % self.random_id())
head = 'tmp=%s ; ' % remote_tmp_dir
#first define a function to clean up and exit
head += 'cleanup(){ '
head += 'test -e $tmp/a && rm $tmp/a >/dev/null 2>&1; '
head += 'test -e $tmp/b && rm $tmp/b >/dev/null 2>&1; '
head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
head += 'test -e $tmp && rmdir $tmp >/dev/null 2>&1; '
head += 'exit $1; }; '
tail = []
#create tmp_RANDOM dir and file a
cmd = 'test -e $tmp || mkdir $tmp; touch $tmp/a; '
tail.append(cmd)
#try to create hardlink b from a
cmd = 'echo \"cp -aRl SOURCE DEST\"; cp -aRl $tmp/a $tmp/b >/dev/null; err_cp=$?; '
cmd += 'test $err_cp -ne 0 && cleanup $err_cp; '
tail.append(cmd)
#list inodes of a and b
cmd = 'ls -i $tmp/a; ls -i $tmp/b; '
tail.append(cmd)
#try to chmod
cmd = 'echo \"chmod u+rw FILE\"; chmod u+rw $tmp/a >/dev/null; err_chmod=$?; '
cmd += 'test $err_chmod -ne 0 && cleanup $err_chmod; '
tail.append(cmd)
#try to find and chmod
cmd = 'echo \"find PATH -type f -exec chmod u-wx \"{}\" \\;\"; '
cmd += 'find $tmp -type f -exec chmod u-wx \"{}\" \\; >/dev/null; err_find=$?; '
cmd += 'test $err_find -ne 0 && cleanup $err_find; '
tail.append(cmd)
#try find suffix '+'
cmd = 'find $tmp -type f -exec chmod u-wx \"{}\" + >/dev/null; err_gnu_find=$?; '
cmd += 'test $err_gnu_find -ne 0 && echo \"gnu_find not supported\"; '
tail.append(cmd)
#try to rm -rf
cmd = 'echo \"rm -rf PATH\"; rm -rf $tmp >/dev/null; err_rm=$?; '
cmd += 'test $err_rm -ne 0 && cleanup $err_rm; '
tail.append(cmd)
#try nice -n 19
if self.nice:
cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try ionice -c2 -n7
if self.ionice:
cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try nocache
if self.nocache:
cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
tail.append(cmd)
#try screen, bash and flock used by smart-remove running in background
if self.config.get_smart_remove_run_remote_in_background(self.profile_id):
cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
cmd += 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
tail.append(cmd)
#if we end up here, everything should be fine
cmd = 'echo \"done\"'
tail.append(cmd)
maxLength = self.config.ssh_max_arg_length(self.profile_id)
additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str))
ssh = ['ssh']
ssh.extend(self.ssh_options + [self.user_host])
ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list))
output = ''
err = ''
returncode = 0
for cmd in tools.splitCommands(tail,
head = head,
maxLength = maxLength - additionalChars):
if cmd.endswith('; '):
cmd += 'echo ""'
c = ssh[:]
c.extend([cmd])
try:
logger.debug('Call command: %s' %' '.join(c), self)
proc = subprocess.Popen(c,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines = True)
ret = proc.communicate()
except OSError as e:
#Argument list too long
if e.errno == 7:
logger.debug('Argument list too log (Python exception)', self)
return maxArg()
else:
raise
logger.debug('Command stdout: %s' %ret[0], self)
logger.debug('Command stderr: %s' %ret[1], self)
logger.debug('Command returncode: %s' %proc.returncode, self)
output += ret[0].strip('\n') + '\n'
err += ret[1].strip('\n') + '\n'
returncode += proc.returncode
if proc.returncode:
break
output_split = output.strip('\n').split('\n')
while True:
if output_split and not output_split[-1]:
output_split = output_split[:-1]
else:
break
if not output_split:
return maxArg()
gnu_find_suffix_support = True
for line in output_split:
if line.startswith('gnu_find not supported'):
gnu_find_suffix_support = False
self.config.set_gnu_find_suffix_support(gnu_find_suffix_support, self.profile_id)
if returncode or not output_split[-1].startswith('done'):
for command in ('cp', 'chmod', 'find', 'rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'):
if output_split[-1].startswith(command):
raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'command' : output_split[-1], 'err' : err})
raise MountException( _('Check commands on host %(host)s returned unknown error:\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'err' : err})
i = 1
inode1 = 'ABC'
inode2 = 'DEF'
for line in output_split:
if line.startswith('cp'):
try:
inode1 = output_split[i].split(' ')[0]
inode2 = output_split[i+1].split(' ')[0]
except IndexError:
pass
if not inode1 == inode2:
raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
i += 1
def checkRemoteCommands(self, retry=False):
"""
Try out all relevant commands used by `Back In Time` on the remote host
to make sure snapshots will be successful with the remote host.
This will also check that hard-links are supported on the remote host.
This check can be disabled with :py:func:`config.Config.sshCheckCommands`
Args:
retry (bool): retry to run the commands if it failed
because the command string was to long
Raises:
exceptions.MountException: if a command is not supported on
remote host or if hard-links are not
supported
"""
if not self.config.sshCheckCommands():
return
logger.debug('Check remote commands', self)
def maxArg():
if retry:
raise MountException(
"Checking commands on remote host didn't return any output. "
"We already checked the maximum argument lenght but it seem like "
"there is an other problem")
logger.warning(
'Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
self)
import sshMaxArg
mid = sshMaxArg.maxArgLength(self.config)
sshMaxArg.reportResult(self.host, mid)
self.config.setSshMaxArgLength(mid, self.profile_id)
return self.checkRemoteCommands(retry=True)
remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.randomId())
remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.randomId())
with tempfile.TemporaryDirectory() as tmp:
tmp_file = os.path.join(tmp, 'a')
with open(tmp_file, 'wt') as f:
f.write('foo')
#check rsync
rsync1 = tools.rsyncPrefix(self.config,
no_perms=False,
progress=False)
rsync1.append(tmp_file)
rsync1.append('%s@%s:"%s"/' %
(self.user, tools.escapeIPv6Address(
self.host), remote_tmp_dir_1))
#check remote rsync hard-link support
rsync2 = tools.rsyncPrefix(self.config,
no_perms=False,
progress=False)
rsync2.append('--link-dest=../%s' %
os.path.basename(remote_tmp_dir_1))
rsync2.append(tmp_file)
rsync2.append('%s@%s:"%s"/' %
(self.user, tools.escapeIPv6Address(
self.host), remote_tmp_dir_2))
for cmd in (rsync1, rsync2):
logger.debug('Check rsync command: %s' % cmd, self)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
out, err = proc.communicate()
if err or proc.returncode:
logger.debug('rsync command returned error: %s' % err,
self)
raise MountException(
_('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions'
) % {
'host': self.host,
'command': cmd,
'err': err
})
#check cp chmod find and rm
head = 'tmp1="%s"; tmp2="%s"; ' % (remote_tmp_dir_1, remote_tmp_dir_2)
#first define a function to clean up and exit
head += 'cleanup(){ '
head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; '
head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; '
head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; '
head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; '
head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; '
head += 'exit $1; }; '
tail = []
#list inodes
cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; '
tail.append(cmd)
#try nice -n 19
if self.nice:
cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try ionice -c2 -n7
if self.ionice:
cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try nocache
if self.nocache:
cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
tail.append(cmd)
#try screen, bash and flock used by smart-remove running in background
if self.config.smartRemoveRunRemoteInBackground(self.profile_id):
cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
tail.append(cmd)
cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
tail.append(cmd)
cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; '
cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; '
tail.append(cmd)
#if we end up here, everything should be fine
cmd = 'echo \"done\"; cleanup 0'
tail.append(cmd)
maxLength = self.config.sshMaxArgLength(self.profile_id)
additionalChars = len('echo ""') + len(
self.config.sshPrefixCmd(self.profile_id, cmd_type=str))
output = ''
err = ''
returncode = 0
for cmd in tools.splitCommands(tail,
head=head,
maxLength=maxLength - additionalChars):
if cmd.endswith('; '):
cmd += 'echo ""'
c = self.config.sshCommand(
cmd=[cmd],
custom_args=['-p', str(self.port), self.user_host],
port=False,
user_host=False,
nice=False,
ionice=False,
profile_id=self.profile_id)
try:
logger.debug('Call command: %s' % ' '.join(c), self)
proc = subprocess.Popen(c,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
ret = proc.communicate()
except OSError as e:
#Argument list too long
if e.errno == 7:
logger.debug('Argument list too log (Python exception)',
self)
return maxArg()
else:
raise
logger.debug('Command stdout: %s' % ret[0], self)
logger.debug('Command stderr: %s' % ret[1], self)
logger.debug('Command returncode: %s' % proc.returncode, self)
output += ret[0].strip('\n') + '\n'
err += ret[1].strip('\n') + '\n'
returncode += proc.returncode
if proc.returncode:
break
output_split = output.strip('\n').split('\n')
while True:
if output_split and not output_split[-1]:
output_split = output_split[:-1]
else:
break
if not output_split:
return maxArg()
if returncode or not output_split[-1].startswith('done'):
for command in ('rm', 'nice', 'ionice', 'nocache', 'screen',
'(flock'):
if output_split[-1].startswith(command):
raise MountException(
_('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions'
) % {
'host': self.host,
'command': output_split[-1],
'err': err
})
raise MountException(
_('Check commands on host %(host)s returned unknown error:\n'
'%(err)s\nLook at \'man backintime\' for further instructions'
) % {
'host': self.host,
'err': err
})
inodes = []
for tmp in (remote_tmp_dir_1, remote_tmp_dir_2):
for line in output_split:
m = re.match(r'^(\d+).*?%s' % tmp, line)
if m:
inodes.append(m.group(1))
logger.debug('remote inodes: ' + ' | '.join(inodes), self)
if len(inodes) == 2 and inodes[0] != inodes[1]:
raise MountException(
_('Remote host %s doesn\'t support hardlinks') % self.host)
def check_remote_commands(self, retry = False):
"""
Try out all relevant commands used by `Back In Time` on the remote host
to make sure snapshots will be successful with the remote host.
This will also check that hard-links are supported on the remote host.
This check can be disabled with :py:func:`config.Config.ssh_check_commands`
Args:
retry (bool): retry to run the commands if it failed
because the command string was to long
Raises:
exceptions.MountException: if a command is not supported on
remote host or if hard-links are not
supported
"""
if not self.config.ssh_check_commands():
return
logger.debug('Check remote commands', self)
def maxArg():
if retry:
raise MountException("Checking commands on remote host didn't return any output. "
"We already checked the maximum argument lenght but it seem like "
"there is an other problem")
logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
self)
import sshMaxArg
mid = sshMaxArg.test_ssh_max_arg(self.user_host)
sshMaxArg.reportResult(self.host, mid)
self.config.set_ssh_max_arg_length(mid, self.profile_id)
return self.check_remote_commands(retry = True)
remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.random_id())
remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.random_id())
with tempfile.TemporaryDirectory() as tmp:
tmp_file = os.path.join(tmp, 'a')
with open(tmp_file, 'wt') as f:
f.write('foo')
#check rsync
rsync1 = tools.get_rsync_prefix(self.config, no_perms = False, progress = False)
rsync1.append(tmp_file)
rsync1.append('%s@%s:%s/' %(self.user, self.host, remote_tmp_dir_1))
#check remote rsync hard-link support
rsync2 = tools.get_rsync_prefix(self.config, no_perms = False, progress = False)
rsync2.append('--link-dest=../%s' %os.path.basename(remote_tmp_dir_1))
rsync2.append(tmp_file)
rsync2.append('%s@%s:%s/' %(self.user, self.host, remote_tmp_dir_2))
for cmd in (rsync1, rsync2):
logger.debug('Check rsync command: %s' %cmd, self)
proc = subprocess.Popen(cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
universal_newlines = True)
out, err = proc.communicate()
if err or proc.returncode:
logger.debug('rsync command returned error: %s' %err, self)
raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'command' : cmd, 'err' : err})
#check cp chmod find and rm
head = 'tmp1=%s; tmp2=%s; ' %(remote_tmp_dir_1, remote_tmp_dir_2)
#first define a function to clean up and exit
head += 'cleanup(){ '
head += 'test -e $tmp1/a && rm $tmp1/a >/dev/null 2>&1; '
head += 'test -e $tmp2/a && rm $tmp2/a >/dev/null 2>&1; '
head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
head += 'test -e $tmp1 && rmdir $tmp1 >/dev/null 2>&1; '
head += 'test -e $tmp2 && rmdir $tmp2 >/dev/null 2>&1; '
head += 'test -n "$tmp3" && test -e $tmp3 && rmdir $tmp3 >/dev/null 2>&1; '
head += 'exit $1; }; '
tail = []
#list inodes
cmd = 'ls -i $tmp1/a; ls -i $tmp2/a; '
tail.append(cmd)
#try nice -n 19
if self.nice:
cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try ionice -c2 -n7
if self.ionice:
cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try nocache
if self.nocache:
cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
tail.append(cmd)
#try screen, bash and flock used by smart-remove running in background
if self.config.get_smart_remove_run_remote_in_background(self.profile_id):
cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
tail.append(cmd)
cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
tail.append(cmd)
cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; '
cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; '
tail.append(cmd)
#if we end up here, everything should be fine
cmd = 'echo \"done\"; cleanup 0'
tail.append(cmd)
maxLength = self.config.ssh_max_arg_length(self.profile_id)
additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str))
ssh = ['ssh']
ssh.extend(self.ssh_options + [self.user_host])
ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list))
output = ''
err = ''
returncode = 0
for cmd in tools.splitCommands(tail,
head = head,
maxLength = maxLength - additionalChars):
if cmd.endswith('; '):
cmd += 'echo ""'
c = ssh[:]
c.extend([cmd])
try:
logger.debug('Call command: %s' %' '.join(c), self)
proc = subprocess.Popen(c,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines = True)
ret = proc.communicate()
except OSError as e:
#Argument list too long
if e.errno == 7:
logger.debug('Argument list too log (Python exception)', self)
return maxArg()
else:
raise
logger.debug('Command stdout: %s' %ret[0], self)
logger.debug('Command stderr: %s' %ret[1], self)
logger.debug('Command returncode: %s' %proc.returncode, self)
output += ret[0].strip('\n') + '\n'
err += ret[1].strip('\n') + '\n'
returncode += proc.returncode
if proc.returncode:
break
output_split = output.strip('\n').split('\n')
while True:
if output_split and not output_split[-1]:
output_split = output_split[:-1]
else:
break
if not output_split:
return maxArg()
if returncode or not output_split[-1].startswith('done'):
for command in ('rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'):
if output_split[-1].startswith(command):
raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'command' : output_split[-1], 'err' : err})
raise MountException( _('Check commands on host %(host)s returned unknown error:\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'err' : err})
inodes = []
for tmp in (remote_tmp_dir_1, remote_tmp_dir_2):
for line in output_split:
m = re.match(r'^(\d+).*?%s' %tmp, line)
if m:
inodes.append(m.group(1))
logger.debug('remote inodes: ' + ' | '.join(inodes), self)
if len(inodes) == 2 and inodes[0] != inodes[1]:
raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
def check_remote_commands(self, retry = False):
"""
try all relevant commands for take_snapshot on remote host.
specialy embedded Linux devices using 'BusyBox' sometimes doesn't
support everything that is need to run backintime.
also check for hardlink-support on remote host.
"""
logger.debug('Check remote commands', self)
def maxArg():
if retry:
raise MountException("Checking commands on remote host didn't return any output. "
"We already checked the maximum argument lenght but it seem like "
"there is an other problem")
logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
self)
import sshMaxArg
mid = sshMaxArg.test_ssh_max_arg(self.user_host)
sshMaxArg.reportResult(self.host, mid)
self.config.set_ssh_max_arg_length(mid, self.profile_id)
return self.check_remote_commands(retry = True)
#check rsync
tmp_file = tempfile.mkstemp()[1]
rsync = tools.get_rsync_prefix( self.config ) + ' --dry-run --chmod=Du+wx %s ' % tmp_file
rsync += '"%s@%s:%s"' % (self.user, self.host, self.path)
logger.debug('Check rsync command: %s' %rsync, self)
#use os.system for compatiblity with snapshots.py
err = os.system(rsync)
if err:
logger.debug('Rsync command returnd error: %s' %err, self)
os.remove(tmp_file)
raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'command' : rsync, 'err' : err})
os.remove(tmp_file)
#check cp chmod find and rm
remote_tmp_dir = os.path.join(self.path, 'tmp_%s' % self.random_id())
head = 'tmp=%s ; ' % remote_tmp_dir
#first define a function to clean up and exit
head += 'cleanup(){ '
head += 'test -e $tmp/a && rm $tmp/a >/dev/null 2>&1; '
head += 'test -e $tmp/b && rm $tmp/b >/dev/null 2>&1; '
head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
head += 'test -e $tmp && rmdir $tmp >/dev/null 2>&1; '
head += 'exit $1; }; '
tail = []
#create tmp_RANDOM dir and file a
cmd = 'test -e $tmp || mkdir $tmp; touch $tmp/a; '
tail.append(cmd)
#try to create hardlink b from a
cmd = 'echo \"cp -aRl SOURCE DEST\"; cp -aRl $tmp/a $tmp/b >/dev/null; err_cp=$?; '
cmd += 'test $err_cp -ne 0 && cleanup $err_cp; '
tail.append(cmd)
#list inodes of a and b
cmd = 'ls -i $tmp/a; ls -i $tmp/b; '
tail.append(cmd)
#try to chmod
cmd = 'echo \"chmod u+rw FILE\"; chmod u+rw $tmp/a >/dev/null; err_chmod=$?; '
cmd += 'test $err_chmod -ne 0 && cleanup $err_chmod; '
tail.append(cmd)
#try to find and chmod
cmd = 'echo \"find PATH -type f -exec chmod u-wx \"{}\" \\;\"; '
cmd += 'find $tmp -type f -exec chmod u-wx \"{}\" \\; >/dev/null; err_find=$?; '
cmd += 'test $err_find -ne 0 && cleanup $err_find; '
tail.append(cmd)
#try find suffix '+'
cmd = 'find $tmp -type f -exec chmod u-wx \"{}\" + >/dev/null; err_gnu_find=$?; '
cmd += 'test $err_gnu_find -ne 0 && echo \"gnu_find not supported\"; '
tail.append(cmd)
#try to rm -rf
cmd = 'echo \"rm -rf PATH\"; rm -rf $tmp >/dev/null; err_rm=$?; '
cmd += 'test $err_rm -ne 0 && cleanup $err_rm; '
tail.append(cmd)
#try nice -n 19
if self.nice:
cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try ionice -c2 -n7
if self.ionice:
cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try nocache
if self.nocache:
cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
tail.append(cmd)
#try screen, bash and flock used by smart-remove running in background
if self.config.get_smart_remove_run_remote_in_background(self.profile_id):
cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
cmd += 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
tail.append(cmd)
#if we end up here, everything should be fine
cmd = 'echo \"done\"'
tail.append(cmd)
maxLength = self.config.ssh_max_arg_length(self.profile_id)
additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str))
ssh = ['ssh']
ssh.extend(self.ssh_options + [self.user_host])
ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list))
output = ''
err = ''
returncode = 0
for cmd in tools.splitCommands(tail,
head = head,
maxLength = maxLength,
additionalChars = additionalChars):
if cmd.endswith('; '):
cmd += 'echo ""'
c = ssh[:]
c.extend([cmd])
try:
logger.debug('Call command: %s' %' '.join(c), self)
proc = subprocess.Popen(c,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines = True)
ret = proc.communicate()
except OSError as e:
#Argument list too long
if e.errno == 7:
logger.debug('Argument list too log (Python exception)', self)
return maxArg()
else:
raise
logger.debug('Command stdout: %s' %ret[0], self)
logger.debug('Command stderr: %s' %ret[1], self)
logger.debug('Command returncode: %s' %proc.returncode, self)
output += ret[0].strip('\n') + '\n'
err += ret[1].strip('\n') + '\n'
returncode += proc.returncode
if proc.returncode:
break
output_split = output.strip('\n').split('\n')
while True:
if output_split and not output_split[-1]:
output_split = output_split[:-1]
else:
break
if not output_split:
return maxArg()
gnu_find_suffix_support = True
for line in output_split:
if line.startswith('gnu_find not supported'):
gnu_find_suffix_support = False
self.config.set_gnu_find_suffix_support(gnu_find_suffix_support, self.profile_id)
if returncode or not output_split[-1].startswith('done'):
for command in ('cp', 'chmod', 'find', 'rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'):
if output_split[-1].startswith(command):
raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'command' : output_split[-1], 'err' : err})
raise MountException( _('Check commands on host %(host)s returned unknown error:\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'err' : err})
i = 1
inode1 = 'ABC'
inode2 = 'DEF'
for line in output_split:
if line.startswith('cp'):
try:
inode1 = output_split[i].split(' ')[0]
inode2 = output_split[i+1].split(' ')[0]
except IndexError:
pass
if not inode1 == inode2:
raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
i += 1
def checkRemoteCommands(self, retry=False):
"""
Try out all relevant commands used by `Back In Time` on the remote host
to make sure snapshots will be successful with the remote host.
This will also check that hard-links are supported on the remote host.
This check can be disabled with :py:func:`config.Config.sshCheckCommands`
Args:
retry (bool): retry to run the commands if it failed
because the command string was to long
Raises:
exceptions.MountException: if a command is not supported on
remote host or if hard-links are not
supported
"""
if not self.config.sshCheckCommands():
return
logger.debug("Check remote commands", self)
def maxArg():
if retry:
raise MountException(
"Checking commands on remote host didn't return any output. "
"We already checked the maximum argument lenght but it seem like "
"there is an other problem"
)
logger.warning(
"Looks like the command was to long for remote SSHd. We will test max arg length now and retry.", self
)
import sshMaxArg
mid = sshMaxArg.maxArgLength(self.config)
sshMaxArg.reportResult(self.host, mid)
self.config.setSshMaxArgLength(mid, self.profile_id)
return self.checkRemoteCommands(retry=True)
remote_tmp_dir_1 = os.path.join(self.path, "tmp_%s" % self.randomId())
remote_tmp_dir_2 = os.path.join(self.path, "tmp_%s" % self.randomId())
with tempfile.TemporaryDirectory() as tmp:
tmp_file = os.path.join(tmp, "a")
with open(tmp_file, "wt") as f:
f.write("foo")
# check rsync
rsync1 = tools.rsyncPrefix(self.config, no_perms=False, progress=False)
rsync1.append(tmp_file)
rsync1.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address(self.host), remote_tmp_dir_1))
# check remote rsync hard-link support
rsync2 = tools.rsyncPrefix(self.config, no_perms=False, progress=False)
rsync2.append("--link-dest=../%s" % os.path.basename(remote_tmp_dir_1))
rsync2.append(tmp_file)
rsync2.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address(self.host), remote_tmp_dir_2))
for cmd in (rsync1, rsync2):
logger.debug("Check rsync command: %s" % cmd, self)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
out, err = proc.communicate()
if err or proc.returncode:
logger.debug("rsync command returned error: %s" % err, self)
raise MountException(
_(
"Remote host %(host)s doesn't support '%(command)s':\n"
"%(err)s\nLook at 'man backintime' for further instructions"
)
% {"host": self.host, "command": cmd, "err": err}
)
# check cp chmod find and rm
head = 'tmp1="%s"; tmp2="%s"; ' % (remote_tmp_dir_1, remote_tmp_dir_2)
# first define a function to clean up and exit
head += "cleanup(){ "
head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; '
head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; '
head += "test -e smr.lock && rm smr.lock >/dev/null 2>&1; "
head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; '
head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; '
head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; '
head += "exit $1; }; "
tail = []
# list inodes
cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; '
tail.append(cmd)
# try nice -n 19
if self.nice:
cmd = 'echo "nice -n 19"; nice -n 19 true >/dev/null; err_nice=$?; '
cmd += "test $err_nice -ne 0 && cleanup $err_nice; "
tail.append(cmd)
# try ionice -c2 -n7
if self.ionice:
cmd = 'echo "ionice -c2 -n7"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
cmd += "test $err_nice -ne 0 && cleanup $err_nice; "
tail.append(cmd)
# try nocache
if self.nocache:
cmd = 'echo "nocache"; nocache true >/dev/null; err_nocache=$?; '
cmd += "test $err_nocache -ne 0 && cleanup $err_nocache; "
tail.append(cmd)
# try screen, bash and flock used by smart-remove running in background
if self.config.smartRemoveRunRemoteInBackground(self.profile_id):
cmd = 'echo "screen -d -m bash -c ..."; screen -d -m bash -c "true" >/dev/null; err_screen=$?; '
cmd += "test $err_screen -ne 0 && cleanup $err_screen; "
tail.append(cmd)
cmd = 'echo "(flock -x 9) 9>smr.lock"; bash -c "(flock -x 9) 9>smr.lock" >/dev/null; err_flock=$?; '
cmd += "test $err_flock -ne 0 && cleanup $err_flock; "
tail.append(cmd)
cmd = 'echo "rmdir \$(mktemp -d)"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; '
cmd += "test $err_rmdir -ne 0 && cleanup $err_rmdir; "
tail.append(cmd)
# if we end up here, everything should be fine
cmd = 'echo "done"; cleanup 0'
tail.append(cmd)
maxLength = self.config.sshMaxArgLength(self.profile_id)
additionalChars = len('echo ""') + len(self.config.sshPrefixCmd(self.profile_id, cmd_type=str))
output = ""
err = ""
returncode = 0
for cmd in tools.splitCommands(tail, head=head, maxLength=maxLength - additionalChars):
if cmd.endswith("; "):
cmd += 'echo ""'
c = self.config.sshCommand(
cmd=[cmd],
custom_args=["-p", str(self.port), self.user_host],
port=False,
cipher=False,
user_host=False,
nice=False,
ionice=False,
profile_id=self.profile_id,
)
try:
logger.debug("Call command: %s" % " ".join(c), self)
proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
ret = proc.communicate()
except OSError as e:
# Argument list too long
if e.errno == 7:
logger.debug("Argument list too log (Python exception)", self)
return maxArg()
else:
raise
logger.debug("Command stdout: %s" % ret[0], self)
logger.debug("Command stderr: %s" % ret[1], self)
logger.debug("Command returncode: %s" % proc.returncode, self)
output += ret[0].strip("\n") + "\n"
err += ret[1].strip("\n") + "\n"
returncode += proc.returncode
if proc.returncode:
break
output_split = output.strip("\n").split("\n")
while True:
if output_split and not output_split[-1]:
output_split = output_split[:-1]
else:
break
if not output_split:
return maxArg()
if returncode or not output_split[-1].startswith("done"):
for command in ("rm", "nice", "ionice", "nocache", "screen", "(flock"):
if output_split[-1].startswith(command):
raise MountException(
_(
"Remote host %(host)s doesn't support '%(command)s':\n"
"%(err)s\nLook at 'man backintime' for further instructions"
)
% {"host": self.host, "command": output_split[-1], "err": err}
)
raise MountException(
_(
"Check commands on host %(host)s returned unknown error:\n"
"%(err)s\nLook at 'man backintime' for further instructions"
)
% {"host": self.host, "err": err}
)
inodes = []
for tmp in (remote_tmp_dir_1, remote_tmp_dir_2):
for line in output_split:
m = re.match(r"^(\d+).*?%s" % tmp, line)
if m:
inodes.append(m.group(1))
logger.debug("remote inodes: " + " | ".join(inodes), self)
if len(inodes) == 2 and inodes[0] != inodes[1]:
raise MountException(_("Remote host %s doesn't support hardlinks") % self.host)