def test_excapeIPv6Address(self):
self.assertEqual(tools.escapeIPv6Address('fd00:0::5'), '[fd00:0::5]')
self.assertEqual(tools.escapeIPv6Address('2001:db8:0:8d3:0:8a2e:70:7344'),
'[2001:db8:0:8d3:0:8a2e:70:7344]')
self.assertEqual(tools.escapeIPv6Address('foo.bar'), 'foo.bar')
self.assertEqual(tools.escapeIPv6Address('192.168.1.1'), '192.168.1.1')
self.assertEqual(tools.escapeIPv6Address('fd00'), 'fd00')
def test_excapeIPv6Address(self):
self.assertEqual(tools.escapeIPv6Address('fd00:0::5'), '[fd00:0::5]')
self.assertEqual(tools.escapeIPv6Address('2001:db8:0:8d3:0:8a2e:70:7344'),
'[2001:db8:0:8d3:0:8a2e:70:7344]')
self.assertEqual(tools.escapeIPv6Address('foo.bar'), 'foo.bar')
self.assertEqual(tools.escapeIPv6Address('192.168.1.1'), '192.168.1.1')
self.assertEqual(tools.escapeIPv6Address('fd00'), 'fd00')
def __init__(self, *args, **kwargs):
#init MountControl
super(SSH, self).__init__(*args, **kwargs)
self.setattrKwargs('user', self.config.sshUser(self.profile_id),
**kwargs)
self.setattrKwargs('host', self.config.sshHost(self.profile_id),
**kwargs)
self.setattrKwargs('port', self.config.sshPort(self.profile_id),
**kwargs)
self.setattrKwargs('path',
self.config.sshSnapshotsPath(self.profile_id),
**kwargs)
self.setattrKwargs('cipher', self.config.sshCipher(self.profile_id),
**kwargs)
self.setattrKwargs('private_key_file',
self.config.sshPrivateKeyFile(self.profile_id),
**kwargs)
self.setattrKwargs('nice',
self.config.niceOnRemote(self.profile_id),
store=False,
**kwargs)
self.setattrKwargs('ionice',
self.config.ioniceOnRemote(self.profile_id),
store=False,
**kwargs)
self.setattrKwargs('nocache',
self.config.nocacheOnRemote(self.profile_id),
store=False,
**kwargs)
self.setattrKwargs('password', None, store=False, **kwargs)
if not self.path:
self.path = './'
self.setDefaultArgs()
# config strings used in ssh-calls
self.user_host_path = '%s@%s:%s' % (
self.user, tools.escapeIPv6Address(self.host), self.path)
self.user_host = '%s@%s' % (self.user, self.host)
self.mountproc = 'sshfs'
self.symlink_subfolder = None
self.log_command = '%s: %s' % (self.mode, self.user_host_path)
self.private_key_fingerprint = tools.sshKeyFingerprint(
self.private_key_file)
if not self.private_key_fingerprint:
logger.warning(
'Couldn\'t get fingerprint for private key %(path)s. '
'Most likely because the public key %(path)s.pub wasn\'t found. '
'Using fallback to private keys path instead. '
'But this can make troubles with passphrase-less keys.' %
{'path': self.private_key_file}, self)
self.private_key_fingerprint = self.private_key_file
self.unlockSshAgent()
def __init__(self, *args, **kwargs):
# init MountControl
super(SSH, self).__init__(*args, **kwargs)
self.setattrKwargs("user", self.config.sshUser(self.profile_id), **kwargs)
self.setattrKwargs("host", self.config.sshHost(self.profile_id), **kwargs)
self.setattrKwargs("port", self.config.sshPort(self.profile_id), **kwargs)
self.setattrKwargs("path", self.config.sshSnapshotsPath(self.profile_id), **kwargs)
self.setattrKwargs("cipher", self.config.sshCipher(self.profile_id), **kwargs)
self.setattrKwargs("private_key_file", self.config.sshPrivateKeyFile(self.profile_id), **kwargs)
self.setattrKwargs("nice", self.config.niceOnRemote(self.profile_id), store=False, **kwargs)
self.setattrKwargs("ionice", self.config.ioniceOnRemote(self.profile_id), store=False, **kwargs)
self.setattrKwargs("nocache", self.config.nocacheOnRemote(self.profile_id), store=False, **kwargs)
self.setattrKwargs("password", None, store=False, **kwargs)
if not self.path:
self.path = "./"
self.setDefaultArgs()
# config strings used in ssh-calls
self.user_host_path = "%s@%s:%s" % (self.user, tools.escapeIPv6Address(self.host), self.path)
self.user_host = "%s@%s" % (self.user, self.host)
self.mountproc = "sshfs"
self.symlink_subfolder = None
self.log_command = "%s: %s" % (self.mode, self.user_host_path)
self.private_key_fingerprint = sshKeyFingerprint(self.private_key_file)
if not self.private_key_fingerprint:
logger.warning(
"Couldn't get fingerprint for private key %(path)s. "
"Most likely because the public key %(path)s.pub wasn't found. "
"Using fallback to private keys path instead. "
"But this can make troubles with passphrase-less keys." % {"path": self.private_key_file},
self,
)
self.private_key_fingerprint = self.private_key_file
self.unlockSshAgent()
def __init__(self, *args, **kwargs):
#init MountControl
super(SSH, self).__init__(*args, **kwargs)
self.setattrKwargs('user', self.config.sshUser(self.profile_id), **kwargs)
self.setattrKwargs('host', self.config.sshHost(self.profile_id), **kwargs)
self.setattrKwargs('port', self.config.sshPort(self.profile_id), **kwargs)
self.setattrKwargs('path', self.config.sshSnapshotsPath(self.profile_id), **kwargs)
self.setattrKwargs('cipher', self.config.sshCipher(self.profile_id), **kwargs)
self.setattrKwargs('private_key_file', self.config.sshPrivateKeyFile(self.profile_id), **kwargs)
self.setattrKwargs('nice', self.config.niceOnRemote(self.profile_id), store = False, **kwargs)
self.setattrKwargs('ionice', self.config.ioniceOnRemote(self.profile_id), store = False, **kwargs)
self.setattrKwargs('nocache', self.config.nocacheOnRemote(self.profile_id), store = False, **kwargs)
self.setattrKwargs('password', None, store = False, **kwargs)
if not self.path:
self.path = './'
self.setDefaultArgs()
# config strings used in ssh-calls
self.user_host_path = '%s@%s:%s' % (self.user, tools.escapeIPv6Address(self.host), self.path)
self.user_host = '%s@%s' % (self.user, self.host)
self.mountproc = 'sshfs'
self.symlink_subfolder = None
self.log_command = '%s: %s' % (self.mode, self.user_host_path)
self.private_key_fingerprint = sshKeyFingerprint(self.private_key_file)
if not self.private_key_fingerprint:
logger.warning('Couldn\'t get fingerprint for private key %(path)s. '
'Most likely because the public key %(path)s.pub wasn\'t found. '
'Using fallback to private keys path instead. '
'But this can make troubles with passphrase-less keys.'
%{'path': self.private_key_file},
self)
self.private_key_fingerprint = self.private_key_file
self.unlockSshAgent()
def checkRemoteCommands(self, retry=False):
"""
Try out all relevant commands used by `Back In Time` on the remote host
to make sure snapshots will be successful with the remote host.
This will also check that hard-links are supported on the remote host.
This check can be disabled with :py:func:`config.Config.sshCheckCommands`
Args:
retry (bool): retry to run the commands if it failed
because the command string was to long
Raises:
exceptions.MountException: if a command is not supported on
remote host or if hard-links are not
supported
"""
if not self.config.sshCheckCommands():
return
logger.debug('Check remote commands', self)
def maxArg():
if retry:
raise MountException(
"Checking commands on remote host didn't return any output. "
"We already checked the maximum argument lenght but it seem like "
"there is an other problem")
logger.warning(
'Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
self)
import sshMaxArg
mid = sshMaxArg.maxArgLength(self.config)
sshMaxArg.reportResult(self.host, mid)
self.config.setSshMaxArgLength(mid, self.profile_id)
return self.checkRemoteCommands(retry=True)
remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.randomId())
remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.randomId())
with tempfile.TemporaryDirectory() as tmp:
tmp_file = os.path.join(tmp, 'a')
with open(tmp_file, 'wt') as f:
f.write('foo')
#check rsync
rsync1 = tools.rsyncPrefix(self.config,
no_perms=False,
progress=False)
rsync1.append(tmp_file)
rsync1.append('%s@%s:"%s"/' %
(self.user, tools.escapeIPv6Address(
self.host), remote_tmp_dir_1))
#check remote rsync hard-link support
rsync2 = tools.rsyncPrefix(self.config,
no_perms=False,
progress=False)
rsync2.append('--link-dest=../%s' %
os.path.basename(remote_tmp_dir_1))
rsync2.append(tmp_file)
rsync2.append('%s@%s:"%s"/' %
(self.user, tools.escapeIPv6Address(
self.host), remote_tmp_dir_2))
for cmd in (rsync1, rsync2):
logger.debug('Check rsync command: %s' % cmd, self)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
out, err = proc.communicate()
if err or proc.returncode:
logger.debug('rsync command returned error: %s' % err,
self)
raise MountException(
_('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions'
) % {
'host': self.host,
'command': cmd,
'err': err
})
#check cp chmod find and rm
head = 'tmp1="%s"; tmp2="%s"; ' % (remote_tmp_dir_1, remote_tmp_dir_2)
#first define a function to clean up and exit
head += 'cleanup(){ '
head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; '
head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; '
head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; '
head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; '
head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; '
head += 'exit $1; }; '
tail = []
#list inodes
cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; '
tail.append(cmd)
#try nice -n 19
if self.nice:
cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try ionice -c2 -n7
if self.ionice:
cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try nocache
if self.nocache:
cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
tail.append(cmd)
#try screen, bash and flock used by smart-remove running in background
if self.config.smartRemoveRunRemoteInBackground(self.profile_id):
cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
tail.append(cmd)
cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
tail.append(cmd)
cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; '
cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; '
tail.append(cmd)
#if we end up here, everything should be fine
cmd = 'echo \"done\"; cleanup 0'
tail.append(cmd)
maxLength = self.config.sshMaxArgLength(self.profile_id)
additionalChars = len('echo ""') + len(
self.config.sshPrefixCmd(self.profile_id, cmd_type=str))
output = ''
err = ''
returncode = 0
for cmd in tools.splitCommands(tail,
head=head,
maxLength=maxLength - additionalChars):
if cmd.endswith('; '):
cmd += 'echo ""'
c = self.config.sshCommand(
cmd=[cmd],
custom_args=['-p', str(self.port), self.user_host],
port=False,
user_host=False,
nice=False,
ionice=False,
profile_id=self.profile_id)
try:
logger.debug('Call command: %s' % ' '.join(c), self)
proc = subprocess.Popen(c,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
ret = proc.communicate()
except OSError as e:
#Argument list too long
if e.errno == 7:
logger.debug('Argument list too log (Python exception)',
self)
return maxArg()
else:
raise
logger.debug('Command stdout: %s' % ret[0], self)
logger.debug('Command stderr: %s' % ret[1], self)
logger.debug('Command returncode: %s' % proc.returncode, self)
output += ret[0].strip('\n') + '\n'
err += ret[1].strip('\n') + '\n'
returncode += proc.returncode
if proc.returncode:
break
output_split = output.strip('\n').split('\n')
while True:
if output_split and not output_split[-1]:
output_split = output_split[:-1]
else:
break
if not output_split:
return maxArg()
if returncode or not output_split[-1].startswith('done'):
for command in ('rm', 'nice', 'ionice', 'nocache', 'screen',
'(flock'):
if output_split[-1].startswith(command):
raise MountException(
_('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions'
) % {
'host': self.host,
'command': output_split[-1],
'err': err
})
raise MountException(
_('Check commands on host %(host)s returned unknown error:\n'
'%(err)s\nLook at \'man backintime\' for further instructions'
) % {
'host': self.host,
'err': err
})
inodes = []
for tmp in (remote_tmp_dir_1, remote_tmp_dir_2):
for line in output_split:
m = re.match(r'^(\d+).*?%s' % tmp, line)
if m:
inodes.append(m.group(1))
logger.debug('remote inodes: ' + ' | '.join(inodes), self)
if len(inodes) == 2 and inodes[0] != inodes[1]:
raise MountException(
_('Remote host %s doesn\'t support hardlinks') % self.host)
def checkRemoteCommands(self, retry = False):
"""
Try out all relevant commands used by `Back In Time` on the remote host
to make sure snapshots will be successful with the remote host.
This will also check that hard-links are supported on the remote host.
This check can be disabled with :py:func:`config.Config.sshCheckCommands`
Args:
retry (bool): retry to run the commands if it failed
because the command string was to long
Raises:
exceptions.MountException: if a command is not supported on
remote host or if hard-links are not
supported
"""
if not self.config.sshCheckCommands():
return
logger.debug('Check remote commands', self)
def maxArg():
if retry:
raise MountException("Checking commands on remote host didn't return any output. "
"We already checked the maximum argument lenght but it seem like "
"there is an other problem")
logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
self)
import sshMaxArg
mid = sshMaxArg.maxArgLength(self.config)
sshMaxArg.reportResult(self.host, mid)
self.config.setSshMaxArgLength(mid, self.profile_id)
return self.checkRemoteCommands(retry = True)
remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.randomId())
remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.randomId())
with tempfile.TemporaryDirectory() as tmp:
tmp_file = os.path.join(tmp, 'a')
with open(tmp_file, 'wt') as f:
f.write('foo')
#check rsync
rsync1 = tools.rsyncPrefix(self.config, no_perms = False, progress = False)
rsync1.append(tmp_file)
rsync1.append('%s@%s:"%s"/' %(self.user,
tools.escapeIPv6Address(self.host),
remote_tmp_dir_1))
#check remote rsync hard-link support
rsync2 = tools.rsyncPrefix(self.config, no_perms = False, progress = False)
rsync2.append('--link-dest=../%s' %os.path.basename(remote_tmp_dir_1))
rsync2.append(tmp_file)
rsync2.append('%s@%s:"%s"/' %(self.user,
tools.escapeIPv6Address(self.host),
remote_tmp_dir_2))
for cmd in (rsync1, rsync2):
logger.debug('Check rsync command: %s' %cmd, self)
proc = subprocess.Popen(cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
universal_newlines = True)
out, err = proc.communicate()
if err or proc.returncode:
logger.debug('rsync command returned error: %s' %err, self)
raise MountException(_('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'command' : cmd, 'err' : err})
#check cp chmod find and rm
head = 'tmp1="%s"; tmp2="%s"; ' %(remote_tmp_dir_1, remote_tmp_dir_2)
#first define a function to clean up and exit
head += 'cleanup(){ '
head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; '
head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; '
head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; '
head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; '
head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; '
head += 'exit $1; }; '
tail = []
#list inodes
cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; '
tail.append(cmd)
#try nice -n 19
if self.nice:
cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try ionice -c2 -n7
if self.ionice:
cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
tail.append(cmd)
#try nocache
if self.nocache:
cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
tail.append(cmd)
#try screen, bash and flock used by smart-remove running in background
if self.config.smartRemoveRunRemoteInBackground(self.profile_id):
cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
tail.append(cmd)
cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
tail.append(cmd)
cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; '
cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; '
tail.append(cmd)
#if we end up here, everything should be fine
cmd = 'echo \"done\"; cleanup 0'
tail.append(cmd)
maxLength = self.config.sshMaxArgLength(self.profile_id)
additionalChars = len('echo ""') + len(self.config.sshPrefixCmd(self.profile_id, cmd_type = str))
output = ''
err = ''
returncode = 0
for cmd in tools.splitCommands(tail,
head = head,
maxLength = maxLength - additionalChars):
if cmd.endswith('; '):
cmd += 'echo ""'
c = self.config.sshCommand(cmd = [cmd],
custom_args = ['-p', str(self.port), self.user_host],
port = False,
cipher = False,
user_host = False,
nice = False,
ionice = False,
profile_id = self.profile_id)
try:
logger.debug('Call command: %s' %' '.join(c), self)
proc = subprocess.Popen(c,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines = True)
ret = proc.communicate()
except OSError as e:
#Argument list too long
if e.errno == 7:
logger.debug('Argument list too log (Python exception)', self)
return maxArg()
else:
raise
logger.debug('Command stdout: %s' %ret[0], self)
logger.debug('Command stderr: %s' %ret[1], self)
logger.debug('Command returncode: %s' %proc.returncode, self)
output += ret[0].strip('\n') + '\n'
err += ret[1].strip('\n') + '\n'
returncode += proc.returncode
if proc.returncode:
break
output_split = output.strip('\n').split('\n')
while True:
if output_split and not output_split[-1]:
output_split = output_split[:-1]
else:
break
if not output_split:
return maxArg()
if returncode or not output_split[-1].startswith('done'):
for command in ('rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'):
if output_split[-1].startswith(command):
raise MountException(_('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'command' : output_split[-1], 'err' : err})
raise MountException(_('Check commands on host %(host)s returned unknown error:\n'
'%(err)s\nLook at \'man backintime\' for further instructions')
% {'host' : self.host, 'err' : err})
inodes = []
for tmp in (remote_tmp_dir_1, remote_tmp_dir_2):
for line in output_split:
m = re.match(r'^(\d+).*?%s' %tmp, line)
if m:
inodes.append(m.group(1))
logger.debug('remote inodes: ' + ' | '.join(inodes), self)
if len(inodes) == 2 and inodes[0] != inodes[1]:
raise MountException(_('Remote host %s doesn\'t support hardlinks') % self.host)
