def test_excapeIPv6Address(self): self.assertEqual(tools.escapeIPv6Address('fd00:0::5'), '[fd00:0::5]') self.assertEqual(tools.escapeIPv6Address('2001:db8:0:8d3:0:8a2e:70:7344'), '[2001:db8:0:8d3:0:8a2e:70:7344]') self.assertEqual(tools.escapeIPv6Address('foo.bar'), 'foo.bar') self.assertEqual(tools.escapeIPv6Address('192.168.1.1'), '192.168.1.1') self.assertEqual(tools.escapeIPv6Address('fd00'), 'fd00')
def __init__(self, *args, **kwargs): #init MountControl super(SSH, self).__init__(*args, **kwargs) self.setattrKwargs('user', self.config.sshUser(self.profile_id), **kwargs) self.setattrKwargs('host', self.config.sshHost(self.profile_id), **kwargs) self.setattrKwargs('port', self.config.sshPort(self.profile_id), **kwargs) self.setattrKwargs('path', self.config.sshSnapshotsPath(self.profile_id), **kwargs) self.setattrKwargs('cipher', self.config.sshCipher(self.profile_id), **kwargs) self.setattrKwargs('private_key_file', self.config.sshPrivateKeyFile(self.profile_id), **kwargs) self.setattrKwargs('nice', self.config.niceOnRemote(self.profile_id), store=False, **kwargs) self.setattrKwargs('ionice', self.config.ioniceOnRemote(self.profile_id), store=False, **kwargs) self.setattrKwargs('nocache', self.config.nocacheOnRemote(self.profile_id), store=False, **kwargs) self.setattrKwargs('password', None, store=False, **kwargs) if not self.path: self.path = './' self.setDefaultArgs() # config strings used in ssh-calls self.user_host_path = '%s@%s:%s' % ( self.user, tools.escapeIPv6Address(self.host), self.path) self.user_host = '%s@%s' % (self.user, self.host) self.mountproc = 'sshfs' self.symlink_subfolder = None self.log_command = '%s: %s' % (self.mode, self.user_host_path) self.private_key_fingerprint = tools.sshKeyFingerprint( self.private_key_file) if not self.private_key_fingerprint: logger.warning( 'Couldn\'t get fingerprint for private key %(path)s. ' 'Most likely because the public key %(path)s.pub wasn\'t found. ' 'Using fallback to private keys path instead. ' 'But this can make troubles with passphrase-less keys.' % {'path': self.private_key_file}, self) self.private_key_fingerprint = self.private_key_file self.unlockSshAgent()
def __init__(self, *args, **kwargs): # init MountControl super(SSH, self).__init__(*args, **kwargs) self.setattrKwargs("user", self.config.sshUser(self.profile_id), **kwargs) self.setattrKwargs("host", self.config.sshHost(self.profile_id), **kwargs) self.setattrKwargs("port", self.config.sshPort(self.profile_id), **kwargs) self.setattrKwargs("path", self.config.sshSnapshotsPath(self.profile_id), **kwargs) self.setattrKwargs("cipher", self.config.sshCipher(self.profile_id), **kwargs) self.setattrKwargs("private_key_file", self.config.sshPrivateKeyFile(self.profile_id), **kwargs) self.setattrKwargs("nice", self.config.niceOnRemote(self.profile_id), store=False, **kwargs) self.setattrKwargs("ionice", self.config.ioniceOnRemote(self.profile_id), store=False, **kwargs) self.setattrKwargs("nocache", self.config.nocacheOnRemote(self.profile_id), store=False, **kwargs) self.setattrKwargs("password", None, store=False, **kwargs) if not self.path: self.path = "./" self.setDefaultArgs() # config strings used in ssh-calls self.user_host_path = "%s@%s:%s" % (self.user, tools.escapeIPv6Address(self.host), self.path) self.user_host = "%s@%s" % (self.user, self.host) self.mountproc = "sshfs" self.symlink_subfolder = None self.log_command = "%s: %s" % (self.mode, self.user_host_path) self.private_key_fingerprint = sshKeyFingerprint(self.private_key_file) if not self.private_key_fingerprint: logger.warning( "Couldn't get fingerprint for private key %(path)s. " "Most likely because the public key %(path)s.pub wasn't found. " "Using fallback to private keys path instead. " "But this can make troubles with passphrase-less keys." % {"path": self.private_key_file}, self, ) self.private_key_fingerprint = self.private_key_file self.unlockSshAgent()
def __init__(self, *args, **kwargs): #init MountControl super(SSH, self).__init__(*args, **kwargs) self.setattrKwargs('user', self.config.sshUser(self.profile_id), **kwargs) self.setattrKwargs('host', self.config.sshHost(self.profile_id), **kwargs) self.setattrKwargs('port', self.config.sshPort(self.profile_id), **kwargs) self.setattrKwargs('path', self.config.sshSnapshotsPath(self.profile_id), **kwargs) self.setattrKwargs('cipher', self.config.sshCipher(self.profile_id), **kwargs) self.setattrKwargs('private_key_file', self.config.sshPrivateKeyFile(self.profile_id), **kwargs) self.setattrKwargs('nice', self.config.niceOnRemote(self.profile_id), store = False, **kwargs) self.setattrKwargs('ionice', self.config.ioniceOnRemote(self.profile_id), store = False, **kwargs) self.setattrKwargs('nocache', self.config.nocacheOnRemote(self.profile_id), store = False, **kwargs) self.setattrKwargs('password', None, store = False, **kwargs) if not self.path: self.path = './' self.setDefaultArgs() # config strings used in ssh-calls self.user_host_path = '%s@%s:%s' % (self.user, tools.escapeIPv6Address(self.host), self.path) self.user_host = '%s@%s' % (self.user, self.host) self.mountproc = 'sshfs' self.symlink_subfolder = None self.log_command = '%s: %s' % (self.mode, self.user_host_path) self.private_key_fingerprint = sshKeyFingerprint(self.private_key_file) if not self.private_key_fingerprint: logger.warning('Couldn\'t get fingerprint for private key %(path)s. ' 'Most likely because the public key %(path)s.pub wasn\'t found. ' 'Using fallback to private keys path instead. ' 'But this can make troubles with passphrase-less keys.' %{'path': self.private_key_file}, self) self.private_key_fingerprint = self.private_key_file self.unlockSshAgent()
def checkRemoteCommands(self, retry=False): """ Try out all relevant commands used by `Back In Time` on the remote host to make sure snapshots will be successful with the remote host. This will also check that hard-links are supported on the remote host. This check can be disabled with :py:func:`config.Config.sshCheckCommands` Args: retry (bool): retry to run the commands if it failed because the command string was to long Raises: exceptions.MountException: if a command is not supported on remote host or if hard-links are not supported """ if not self.config.sshCheckCommands(): return logger.debug('Check remote commands', self) def maxArg(): if retry: raise MountException( "Checking commands on remote host didn't return any output. " "We already checked the maximum argument lenght but it seem like " "there is an other problem") logger.warning( 'Looks like the command was to long for remote SSHd. We will test max arg length now and retry.', self) import sshMaxArg mid = sshMaxArg.maxArgLength(self.config) sshMaxArg.reportResult(self.host, mid) self.config.setSshMaxArgLength(mid, self.profile_id) return self.checkRemoteCommands(retry=True) remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.randomId()) remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.randomId()) with tempfile.TemporaryDirectory() as tmp: tmp_file = os.path.join(tmp, 'a') with open(tmp_file, 'wt') as f: f.write('foo') #check rsync rsync1 = tools.rsyncPrefix(self.config, no_perms=False, progress=False) rsync1.append(tmp_file) rsync1.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address( self.host), remote_tmp_dir_1)) #check remote rsync hard-link support rsync2 = tools.rsyncPrefix(self.config, no_perms=False, progress=False) rsync2.append('--link-dest=../%s' % os.path.basename(remote_tmp_dir_1)) rsync2.append(tmp_file) rsync2.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address( self.host), remote_tmp_dir_2)) for cmd in (rsync1, rsync2): logger.debug('Check rsync command: %s' % cmd, self) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = proc.communicate() if err or proc.returncode: logger.debug('rsync command returned error: %s' % err, self) raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions' ) % { 'host': self.host, 'command': cmd, 'err': err }) #check cp chmod find and rm head = 'tmp1="%s"; tmp2="%s"; ' % (remote_tmp_dir_1, remote_tmp_dir_2) #first define a function to clean up and exit head += 'cleanup(){ ' head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; ' head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; ' head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; ' head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; ' head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; ' head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; ' head += 'exit $1; }; ' tail = [] #list inodes cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; ' tail.append(cmd) #try nice -n 19 if self.nice: cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try ionice -c2 -n7 if self.ionice: cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try nocache if self.nocache: cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; ' cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; ' tail.append(cmd) #try screen, bash and flock used by smart-remove running in background if self.config.smartRemoveRunRemoteInBackground(self.profile_id): cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; ' cmd += 'test $err_screen -ne 0 && cleanup $err_screen; ' tail.append(cmd) cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; ' cmd += 'test $err_flock -ne 0 && cleanup $err_flock; ' tail.append(cmd) cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; ' cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; ' tail.append(cmd) #if we end up here, everything should be fine cmd = 'echo \"done\"; cleanup 0' tail.append(cmd) maxLength = self.config.sshMaxArgLength(self.profile_id) additionalChars = len('echo ""') + len( self.config.sshPrefixCmd(self.profile_id, cmd_type=str)) output = '' err = '' returncode = 0 for cmd in tools.splitCommands(tail, head=head, maxLength=maxLength - additionalChars): if cmd.endswith('; '): cmd += 'echo ""' c = self.config.sshCommand( cmd=[cmd], custom_args=['-p', str(self.port), self.user_host], port=False, user_host=False, nice=False, ionice=False, profile_id=self.profile_id) try: logger.debug('Call command: %s' % ' '.join(c), self) proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) ret = proc.communicate() except OSError as e: #Argument list too long if e.errno == 7: logger.debug('Argument list too log (Python exception)', self) return maxArg() else: raise logger.debug('Command stdout: %s' % ret[0], self) logger.debug('Command stderr: %s' % ret[1], self) logger.debug('Command returncode: %s' % proc.returncode, self) output += ret[0].strip('\n') + '\n' err += ret[1].strip('\n') + '\n' returncode += proc.returncode if proc.returncode: break output_split = output.strip('\n').split('\n') while True: if output_split and not output_split[-1]: output_split = output_split[:-1] else: break if not output_split: return maxArg() if returncode or not output_split[-1].startswith('done'): for command in ('rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'): if output_split[-1].startswith(command): raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions' ) % { 'host': self.host, 'command': output_split[-1], 'err': err }) raise MountException( _('Check commands on host %(host)s returned unknown error:\n' '%(err)s\nLook at \'man backintime\' for further instructions' ) % { 'host': self.host, 'err': err }) inodes = [] for tmp in (remote_tmp_dir_1, remote_tmp_dir_2): for line in output_split: m = re.match(r'^(\d+).*?%s' % tmp, line) if m: inodes.append(m.group(1)) logger.debug('remote inodes: ' + ' | '.join(inodes), self) if len(inodes) == 2 and inodes[0] != inodes[1]: raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
def checkRemoteCommands(self, retry = False): """ Try out all relevant commands used by `Back In Time` on the remote host to make sure snapshots will be successful with the remote host. This will also check that hard-links are supported on the remote host. This check can be disabled with :py:func:`config.Config.sshCheckCommands` Args: retry (bool): retry to run the commands if it failed because the command string was to long Raises: exceptions.MountException: if a command is not supported on remote host or if hard-links are not supported """ if not self.config.sshCheckCommands(): return logger.debug('Check remote commands', self) def maxArg(): if retry: raise MountException("Checking commands on remote host didn't return any output. " "We already checked the maximum argument lenght but it seem like " "there is an other problem") logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.', self) import sshMaxArg mid = sshMaxArg.maxArgLength(self.config) sshMaxArg.reportResult(self.host, mid) self.config.setSshMaxArgLength(mid, self.profile_id) return self.checkRemoteCommands(retry = True) remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.randomId()) remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.randomId()) with tempfile.TemporaryDirectory() as tmp: tmp_file = os.path.join(tmp, 'a') with open(tmp_file, 'wt') as f: f.write('foo') #check rsync rsync1 = tools.rsyncPrefix(self.config, no_perms = False, progress = False) rsync1.append(tmp_file) rsync1.append('%s@%s:"%s"/' %(self.user, tools.escapeIPv6Address(self.host), remote_tmp_dir_1)) #check remote rsync hard-link support rsync2 = tools.rsyncPrefix(self.config, no_perms = False, progress = False) rsync2.append('--link-dest=../%s' %os.path.basename(remote_tmp_dir_1)) rsync2.append(tmp_file) rsync2.append('%s@%s:"%s"/' %(self.user, tools.escapeIPv6Address(self.host), remote_tmp_dir_2)) for cmd in (rsync1, rsync2): logger.debug('Check rsync command: %s' %cmd, self) proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True) out, err = proc.communicate() if err or proc.returncode: logger.debug('rsync command returned error: %s' %err, self) raise MountException(_('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'command' : cmd, 'err' : err}) #check cp chmod find and rm head = 'tmp1="%s"; tmp2="%s"; ' %(remote_tmp_dir_1, remote_tmp_dir_2) #first define a function to clean up and exit head += 'cleanup(){ ' head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; ' head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; ' head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; ' head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; ' head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; ' head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; ' head += 'exit $1; }; ' tail = [] #list inodes cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; ' tail.append(cmd) #try nice -n 19 if self.nice: cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try ionice -c2 -n7 if self.ionice: cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try nocache if self.nocache: cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; ' cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; ' tail.append(cmd) #try screen, bash and flock used by smart-remove running in background if self.config.smartRemoveRunRemoteInBackground(self.profile_id): cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; ' cmd += 'test $err_screen -ne 0 && cleanup $err_screen; ' tail.append(cmd) cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; ' cmd += 'test $err_flock -ne 0 && cleanup $err_flock; ' tail.append(cmd) cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; ' cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; ' tail.append(cmd) #if we end up here, everything should be fine cmd = 'echo \"done\"; cleanup 0' tail.append(cmd) maxLength = self.config.sshMaxArgLength(self.profile_id) additionalChars = len('echo ""') + len(self.config.sshPrefixCmd(self.profile_id, cmd_type = str)) output = '' err = '' returncode = 0 for cmd in tools.splitCommands(tail, head = head, maxLength = maxLength - additionalChars): if cmd.endswith('; '): cmd += 'echo ""' c = self.config.sshCommand(cmd = [cmd], custom_args = ['-p', str(self.port), self.user_host], port = False, cipher = False, user_host = False, nice = False, ionice = False, profile_id = self.profile_id) try: logger.debug('Call command: %s' %' '.join(c), self) proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines = True) ret = proc.communicate() except OSError as e: #Argument list too long if e.errno == 7: logger.debug('Argument list too log (Python exception)', self) return maxArg() else: raise logger.debug('Command stdout: %s' %ret[0], self) logger.debug('Command stderr: %s' %ret[1], self) logger.debug('Command returncode: %s' %proc.returncode, self) output += ret[0].strip('\n') + '\n' err += ret[1].strip('\n') + '\n' returncode += proc.returncode if proc.returncode: break output_split = output.strip('\n').split('\n') while True: if output_split and not output_split[-1]: output_split = output_split[:-1] else: break if not output_split: return maxArg() if returncode or not output_split[-1].startswith('done'): for command in ('rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'): if output_split[-1].startswith(command): raise MountException(_('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'command' : output_split[-1], 'err' : err}) raise MountException(_('Check commands on host %(host)s returned unknown error:\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'err' : err}) inodes = [] for tmp in (remote_tmp_dir_1, remote_tmp_dir_2): for line in output_split: m = re.match(r'^(\d+).*?%s' %tmp, line) if m: inodes.append(m.group(1)) logger.debug('remote inodes: ' + ' | '.join(inodes), self) if len(inodes) == 2 and inodes[0] != inodes[1]: raise MountException(_('Remote host %s doesn\'t support hardlinks') % self.host)