Beispiel #1
0
    def test_splitCommands(self):
        ret = list(
            tools.splitCommands(['echo foo;'],
                                head='echo start;',
                                tail='echo end',
                                maxLength=40))
        self.assertEqual(len(ret), 1)
        self.assertEqual(ret[0], 'echo start;echo foo;echo end')

        ret = list(
            tools.splitCommands(['echo foo;'] * 3,
                                head='echo start;',
                                tail='echo end',
                                maxLength=40))
        self.assertEqual(len(ret), 2)
        self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo end')
        self.assertEqual(ret[1], 'echo start;echo foo;echo end')

        ret = list(
            tools.splitCommands(['echo foo;'] * 3,
                                head='echo start;',
                                tail='echo end',
                                maxLength=0))
        self.assertEqual(len(ret), 1)
        self.assertEqual(ret[0],
                         'echo start;echo foo;echo foo;echo foo;echo end')

        ret = list(
            tools.splitCommands(['echo foo;'] * 3,
                                head='echo start;',
                                tail='echo end',
                                maxLength=-10))
        self.assertEqual(len(ret), 1)
        self.assertEqual(ret[0],
                         'echo start;echo foo;echo foo;echo foo;echo end')
Beispiel #2
0
    def test_splitCommands(self):
        ret = list(tools.splitCommands(['echo foo;'],
                                       head = 'echo start;',
                                       tail = 'echo end',
                                       maxLength = 40))
        self.assertEqual(len(ret), 1)
        self.assertEqual(ret[0], 'echo start;echo foo;echo end')

        ret = list(tools.splitCommands(['echo foo;']*3,
                                       head = 'echo start;',
                                       tail = 'echo end',
                                       maxLength = 40))
        self.assertEqual(len(ret), 2)
        self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo end')
        self.assertEqual(ret[1], 'echo start;echo foo;echo end')

        ret = list(tools.splitCommands(['echo foo;']*3,
                                       head = 'echo start;',
                                       tail = 'echo end',
                                       maxLength = 0))
        self.assertEqual(len(ret), 1)
        self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo foo;echo end')

        ret = list(tools.splitCommands(['echo foo;']*3,
                                       head = 'echo start;',
                                       tail = 'echo end',
                                       maxLength = -10))
        self.assertEqual(len(ret), 1)
        self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo foo;echo end')
Beispiel #3
0
    def check_remote_commands(self, retry = False):
        """
        try all relevant commands for take_snapshot on remote host.
        specialy embedded Linux devices using 'BusyBox' sometimes doesn't
        support everything that is need to run backintime.
        also check for hardlink-support on remote host.
        """
        logger.debug('Check remote commands', self)
        def maxArg():
            if retry:
                raise MountException("Checking commands on remote host didn't return any output. "
                                     "We already checked the maximum argument lenght but it seem like "
                                     "there is an other problem")
            logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
                           self)
            import sshMaxArg
            mid = sshMaxArg.test_ssh_max_arg(self.user_host)
            sshMaxArg.reportResult(self.host, mid)
            self.config.set_ssh_max_arg_length(mid, self.profile_id)
            return self.check_remote_commands(retry = True)

        #check rsync
        tmp_file = tempfile.mkstemp()[1]
        rsync = tools.get_rsync_prefix( self.config ) + ' --dry-run --chmod=Du+wx %s ' % tmp_file
        rsync += '"%s@%s:%s"' % (self.user, self.host, self.path)
        logger.debug('Check rsync command: %s' %rsync, self)

        #use os.system for compatiblity with snapshots.py
        err = os.system(rsync)
        if err:
            logger.debug('Rsync command returnd error: %s' %err, self)
            os.remove(tmp_file)
            raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
                                    '%(err)s\nLook at \'man backintime\' for further instructions')
                                    % {'host' : self.host, 'command' : rsync, 'err' : err})
        os.remove(tmp_file)

        #check cp chmod find and rm
        remote_tmp_dir = os.path.join(self.path, 'tmp_%s' % self.random_id())
        head  = 'tmp=%s ; ' % remote_tmp_dir
        #first define a function to clean up and exit
        head += 'cleanup(){ '
        head += 'test -e $tmp/a && rm $tmp/a >/dev/null 2>&1; '
        head += 'test -e $tmp/b && rm $tmp/b >/dev/null 2>&1; '
        head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
        head += 'test -e $tmp && rmdir $tmp >/dev/null 2>&1; '
        head += 'exit $1; }; '
        tail = []
        #create tmp_RANDOM dir and file a
        cmd  = 'test -e $tmp || mkdir $tmp; touch $tmp/a; '
        tail.append(cmd)

        #try to create hardlink b from a
        cmd  = 'echo \"cp -aRl SOURCE DEST\"; cp -aRl $tmp/a $tmp/b >/dev/null; err_cp=$?; '
        cmd += 'test $err_cp -ne 0 && cleanup $err_cp; '
        tail.append(cmd)
        #list inodes of a and b
        cmd  = 'ls -i $tmp/a; ls -i $tmp/b; '
        tail.append(cmd)
        #try to chmod
        cmd  = 'echo \"chmod u+rw FILE\"; chmod u+rw $tmp/a >/dev/null; err_chmod=$?; '
        cmd += 'test $err_chmod -ne 0 && cleanup $err_chmod; '
        tail.append(cmd)
        #try to find and chmod
        cmd  = 'echo \"find PATH -type f -exec chmod u-wx \"{}\" \\;\"; '
        cmd += 'find $tmp -type f -exec chmod u-wx \"{}\" \\; >/dev/null; err_find=$?; '
        cmd += 'test $err_find -ne 0 && cleanup $err_find; '
        tail.append(cmd)
        #try find suffix '+'
        cmd  = 'find $tmp -type f -exec chmod u-wx \"{}\" + >/dev/null; err_gnu_find=$?; '
        cmd += 'test $err_gnu_find -ne 0 && echo \"gnu_find not supported\"; '
        tail.append(cmd)
        #try to rm -rf
        cmd  = 'echo \"rm -rf PATH\"; rm -rf $tmp >/dev/null; err_rm=$?; '
        cmd += 'test $err_rm -ne 0 && cleanup $err_rm; '
        tail.append(cmd)
        #try nice -n 19
        if self.nice:
            cmd  = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
            cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
            tail.append(cmd)
        #try ionice -c2 -n7
        if self.ionice:
            cmd  = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
            cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
            tail.append(cmd)
        #try nocache
        if self.nocache:
            cmd  = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
            cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
            tail.append(cmd)
        #try screen, bash and flock used by smart-remove running in background
        if self.config.get_smart_remove_run_remote_in_background(self.profile_id):
            cmd  = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
            cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
            cmd += 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
            cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
            tail.append(cmd)
        #if we end up here, everything should be fine
        cmd = 'echo \"done\"'
        tail.append(cmd)

        maxLength = self.config.ssh_max_arg_length(self.profile_id)
        additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str))

        ssh = ['ssh']
        ssh.extend(self.ssh_options + [self.user_host])
        ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list))
        output = ''
        err = ''
        returncode = 0
        for cmd in tools.splitCommands(tail,
                                       head = head,
                                       maxLength = maxLength - additionalChars):
            if cmd.endswith('; '):
                cmd += 'echo ""'
            c = ssh[:]
            c.extend([cmd])
            try:
                logger.debug('Call command: %s' %' '.join(c), self)
                proc = subprocess.Popen(c,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        universal_newlines = True)
                ret = proc.communicate()
            except OSError as e:
                #Argument list too long
                if e.errno == 7:
                    logger.debug('Argument list too log (Python exception)', self)
                    return maxArg()
                else:
                    raise
            logger.debug('Command stdout: %s' %ret[0], self)
            logger.debug('Command stderr: %s' %ret[1], self)
            logger.debug('Command returncode: %s' %proc.returncode, self)
            output += ret[0].strip('\n') + '\n'
            err    += ret[1].strip('\n') + '\n'
            returncode += proc.returncode
            if proc.returncode:
                break

        output_split = output.strip('\n').split('\n')

        while True:
            if output_split and not output_split[-1]:
                output_split = output_split[:-1]
            else:
                break

        if not output_split:
            return maxArg()

        gnu_find_suffix_support = True
        for line in output_split:
            if line.startswith('gnu_find not supported'):
                gnu_find_suffix_support = False
        self.config.set_gnu_find_suffix_support(gnu_find_suffix_support, self.profile_id)

        if returncode or not output_split[-1].startswith('done'):
            for command in ('cp', 'chmod', 'find', 'rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'):
                if output_split[-1].startswith(command):
                    raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
                                            '%(err)s\nLook at \'man backintime\' for further instructions')
                                            % {'host' : self.host, 'command' : output_split[-1], 'err' : err})
            raise MountException( _('Check commands on host %(host)s returned unknown error:\n'
                                    '%(err)s\nLook at \'man backintime\' for further instructions')
                                    % {'host' : self.host, 'err' : err})

        i = 1
        inode1 = 'ABC'
        inode2 = 'DEF'
        for line in output_split:
            if line.startswith('cp'):
                try:
                    inode1 = output_split[i].split(' ')[0]
                    inode2 = output_split[i+1].split(' ')[0]
                except IndexError:
                    pass
                if not inode1 == inode2:
                    raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
            i += 1
Beispiel #4
0
    def checkRemoteCommands(self, retry=False):
        """
        Try out all relevant commands used by `Back In Time` on the remote host
        to make sure snapshots will be successful with the remote host.
        This will also check that hard-links are supported on the remote host.

        This check can be disabled with :py:func:`config.Config.sshCheckCommands`

        Args:
            retry (bool):               retry to run the commands if it failed
                                        because the command string was to long

        Raises:
            exceptions.MountException:  if a command is not supported on
                                        remote host or if hard-links are not
                                        supported
        """
        if not self.config.sshCheckCommands():
            return
        logger.debug('Check remote commands', self)

        def maxArg():
            if retry:
                raise MountException(
                    "Checking commands on remote host didn't return any output. "
                    "We already checked the maximum argument lenght but it seem like "
                    "there is an other problem")
            logger.warning(
                'Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
                self)
            import sshMaxArg
            mid = sshMaxArg.maxArgLength(self.config)
            sshMaxArg.reportResult(self.host, mid)
            self.config.setSshMaxArgLength(mid, self.profile_id)
            return self.checkRemoteCommands(retry=True)

        remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.randomId())
        remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.randomId())
        with tempfile.TemporaryDirectory() as tmp:
            tmp_file = os.path.join(tmp, 'a')
            with open(tmp_file, 'wt') as f:
                f.write('foo')

            #check rsync
            rsync1 = tools.rsyncPrefix(self.config,
                                       no_perms=False,
                                       progress=False)
            rsync1.append(tmp_file)
            rsync1.append('%s@%s:"%s"/' %
                          (self.user, tools.escapeIPv6Address(
                              self.host), remote_tmp_dir_1))

            #check remote rsync hard-link support
            rsync2 = tools.rsyncPrefix(self.config,
                                       no_perms=False,
                                       progress=False)
            rsync2.append('--link-dest=../%s' %
                          os.path.basename(remote_tmp_dir_1))
            rsync2.append(tmp_file)
            rsync2.append('%s@%s:"%s"/' %
                          (self.user, tools.escapeIPv6Address(
                              self.host), remote_tmp_dir_2))

            for cmd in (rsync1, rsync2):
                logger.debug('Check rsync command: %s' % cmd, self)

                proc = subprocess.Popen(cmd,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        universal_newlines=True)
                out, err = proc.communicate()
                if err or proc.returncode:
                    logger.debug('rsync command returned error: %s' % err,
                                 self)
                    raise MountException(
                        _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
                          '%(err)s\nLook at \'man backintime\' for further instructions'
                          ) % {
                              'host': self.host,
                              'command': cmd,
                              'err': err
                          })

        #check cp chmod find and rm
        head = 'tmp1="%s"; tmp2="%s"; ' % (remote_tmp_dir_1, remote_tmp_dir_2)
        #first define a function to clean up and exit
        head += 'cleanup(){ '
        head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; '
        head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; '
        head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
        head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; '
        head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; '
        head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; '
        head += 'exit $1; }; '
        tail = []

        #list inodes
        cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; '
        tail.append(cmd)
        #try nice -n 19
        if self.nice:
            cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
            cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
            tail.append(cmd)
        #try ionice -c2 -n7
        if self.ionice:
            cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
            cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
            tail.append(cmd)
        #try nocache
        if self.nocache:
            cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
            cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
            tail.append(cmd)
        #try screen, bash and flock used by smart-remove running in background
        if self.config.smartRemoveRunRemoteInBackground(self.profile_id):
            cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
            cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
            tail.append(cmd)
            cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
            cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
            tail.append(cmd)
            cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; '
            cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; '
            tail.append(cmd)
        #if we end up here, everything should be fine
        cmd = 'echo \"done\"; cleanup 0'
        tail.append(cmd)

        maxLength = self.config.sshMaxArgLength(self.profile_id)
        additionalChars = len('echo ""') + len(
            self.config.sshPrefixCmd(self.profile_id, cmd_type=str))

        output = ''
        err = ''
        returncode = 0
        for cmd in tools.splitCommands(tail,
                                       head=head,
                                       maxLength=maxLength - additionalChars):
            if cmd.endswith('; '):
                cmd += 'echo ""'
            c = self.config.sshCommand(
                cmd=[cmd],
                custom_args=['-p', str(self.port), self.user_host],
                port=False,
                user_host=False,
                nice=False,
                ionice=False,
                profile_id=self.profile_id)
            try:
                logger.debug('Call command: %s' % ' '.join(c), self)
                proc = subprocess.Popen(c,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        universal_newlines=True)
                ret = proc.communicate()
            except OSError as e:
                #Argument list too long
                if e.errno == 7:
                    logger.debug('Argument list too log (Python exception)',
                                 self)
                    return maxArg()
                else:
                    raise
            logger.debug('Command stdout: %s' % ret[0], self)
            logger.debug('Command stderr: %s' % ret[1], self)
            logger.debug('Command returncode: %s' % proc.returncode, self)
            output += ret[0].strip('\n') + '\n'
            err += ret[1].strip('\n') + '\n'
            returncode += proc.returncode
            if proc.returncode:
                break

        output_split = output.strip('\n').split('\n')

        while True:
            if output_split and not output_split[-1]:
                output_split = output_split[:-1]
            else:
                break

        if not output_split:
            return maxArg()

        if returncode or not output_split[-1].startswith('done'):
            for command in ('rm', 'nice', 'ionice', 'nocache', 'screen',
                            '(flock'):
                if output_split[-1].startswith(command):
                    raise MountException(
                        _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
                          '%(err)s\nLook at \'man backintime\' for further instructions'
                          ) % {
                              'host': self.host,
                              'command': output_split[-1],
                              'err': err
                          })
            raise MountException(
                _('Check commands on host %(host)s returned unknown error:\n'
                  '%(err)s\nLook at \'man backintime\' for further instructions'
                  ) % {
                      'host': self.host,
                      'err': err
                  })

        inodes = []
        for tmp in (remote_tmp_dir_1, remote_tmp_dir_2):
            for line in output_split:
                m = re.match(r'^(\d+).*?%s' % tmp, line)
                if m:
                    inodes.append(m.group(1))

        logger.debug('remote inodes: ' + ' | '.join(inodes), self)
        if len(inodes) == 2 and inodes[0] != inodes[1]:
            raise MountException(
                _('Remote host %s doesn\'t support hardlinks') % self.host)
Beispiel #5
0
    def check_remote_commands(self, retry = False):
        """
        Try out all relevant commands used by `Back In Time` on the remote host
        to make sure snapshots will be successful with the remote host.
        This will also check that hard-links are supported on the remote host.

        This check can be disabled with :py:func:`config.Config.ssh_check_commands`

        Args:
            retry (bool):               retry to run the commands if it failed
                                        because the command string was to long

        Raises:
            exceptions.MountException:  if a command is not supported on
                                        remote host or if hard-links are not
                                        supported
        """
        if not self.config.ssh_check_commands():
            return
        logger.debug('Check remote commands', self)
        def maxArg():
            if retry:
                raise MountException("Checking commands on remote host didn't return any output. "
                                     "We already checked the maximum argument lenght but it seem like "
                                     "there is an other problem")
            logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
                           self)
            import sshMaxArg
            mid = sshMaxArg.test_ssh_max_arg(self.user_host)
            sshMaxArg.reportResult(self.host, mid)
            self.config.set_ssh_max_arg_length(mid, self.profile_id)
            return self.check_remote_commands(retry = True)

        remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.random_id())
        remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.random_id())
        with tempfile.TemporaryDirectory() as tmp:
            tmp_file = os.path.join(tmp, 'a')
            with open(tmp_file, 'wt') as f:
                f.write('foo')

            #check rsync
            rsync1 =  tools.get_rsync_prefix(self.config, no_perms = False, progress = False)
            rsync1.append(tmp_file)
            rsync1.append('%s@%s:%s/' %(self.user, self.host, remote_tmp_dir_1))

            #check remote rsync hard-link support
            rsync2 =  tools.get_rsync_prefix(self.config, no_perms = False, progress = False)
            rsync2.append('--link-dest=../%s' %os.path.basename(remote_tmp_dir_1))
            rsync2.append(tmp_file)
            rsync2.append('%s@%s:%s/' %(self.user, self.host, remote_tmp_dir_2))

            for cmd in (rsync1, rsync2):
                logger.debug('Check rsync command: %s' %cmd, self)

                proc = subprocess.Popen(cmd,
                                        stdout = subprocess.PIPE,
                                        stderr = subprocess.PIPE,
                                        universal_newlines = True)
                out, err = proc.communicate()
                if err or proc.returncode:
                    logger.debug('rsync command returned error: %s' %err, self)
                    raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
                                            '%(err)s\nLook at \'man backintime\' for further instructions')
                                            % {'host' : self.host, 'command' : cmd, 'err' : err})

        #check cp chmod find and rm
        head  = 'tmp1=%s; tmp2=%s; ' %(remote_tmp_dir_1, remote_tmp_dir_2)
        #first define a function to clean up and exit
        head += 'cleanup(){ '
        head += 'test -e $tmp1/a && rm $tmp1/a >/dev/null 2>&1; '
        head += 'test -e $tmp2/a && rm $tmp2/a >/dev/null 2>&1; '
        head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
        head += 'test -e $tmp1 && rmdir $tmp1 >/dev/null 2>&1; '
        head += 'test -e $tmp2 && rmdir $tmp2 >/dev/null 2>&1; '
        head += 'test -n "$tmp3" && test -e $tmp3 && rmdir $tmp3 >/dev/null 2>&1; '
        head += 'exit $1; }; '
        tail = []

        #list inodes
        cmd  = 'ls -i $tmp1/a; ls -i $tmp2/a; '
        tail.append(cmd)
        #try nice -n 19
        if self.nice:
            cmd  = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
            cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
            tail.append(cmd)
        #try ionice -c2 -n7
        if self.ionice:
            cmd  = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
            cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
            tail.append(cmd)
        #try nocache
        if self.nocache:
            cmd  = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
            cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
            tail.append(cmd)
        #try screen, bash and flock used by smart-remove running in background
        if self.config.get_smart_remove_run_remote_in_background(self.profile_id):
            cmd  = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
            cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
            tail.append(cmd)
            cmd  = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
            cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
            tail.append(cmd)
            cmd  = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; '
            cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; '
            tail.append(cmd)
        #if we end up here, everything should be fine
        cmd = 'echo \"done\"; cleanup 0'
        tail.append(cmd)

        maxLength = self.config.ssh_max_arg_length(self.profile_id)
        additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str))

        ssh = ['ssh']
        ssh.extend(self.ssh_options + [self.user_host])
        ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list))
        output = ''
        err = ''
        returncode = 0
        for cmd in tools.splitCommands(tail,
                                       head = head,
                                       maxLength = maxLength - additionalChars):
            if cmd.endswith('; '):
                cmd += 'echo ""'
            c = ssh[:]
            c.extend([cmd])
            try:
                logger.debug('Call command: %s' %' '.join(c), self)
                proc = subprocess.Popen(c,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        universal_newlines = True)
                ret = proc.communicate()
            except OSError as e:
                #Argument list too long
                if e.errno == 7:
                    logger.debug('Argument list too log (Python exception)', self)
                    return maxArg()
                else:
                    raise
            logger.debug('Command stdout: %s' %ret[0], self)
            logger.debug('Command stderr: %s' %ret[1], self)
            logger.debug('Command returncode: %s' %proc.returncode, self)
            output += ret[0].strip('\n') + '\n'
            err    += ret[1].strip('\n') + '\n'
            returncode += proc.returncode
            if proc.returncode:
                break

        output_split = output.strip('\n').split('\n')

        while True:
            if output_split and not output_split[-1]:
                output_split = output_split[:-1]
            else:
                break

        if not output_split:
            return maxArg()

        if returncode or not output_split[-1].startswith('done'):
            for command in ('rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'):
                if output_split[-1].startswith(command):
                    raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
                                            '%(err)s\nLook at \'man backintime\' for further instructions')
                                            % {'host' : self.host, 'command' : output_split[-1], 'err' : err})
            raise MountException( _('Check commands on host %(host)s returned unknown error:\n'
                                    '%(err)s\nLook at \'man backintime\' for further instructions')
                                    % {'host' : self.host, 'err' : err})

        inodes = []
        for tmp in (remote_tmp_dir_1, remote_tmp_dir_2):
            for line in output_split:
                m = re.match(r'^(\d+).*?%s' %tmp, line)
                if m:
                    inodes.append(m.group(1))

        logger.debug('remote inodes: ' + ' | '.join(inodes), self)
        if len(inodes) == 2 and inodes[0] != inodes[1]:
            raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
Beispiel #6
0
    def check_remote_commands(self, retry = False):
        """
        try all relevant commands for take_snapshot on remote host.
        specialy embedded Linux devices using 'BusyBox' sometimes doesn't
        support everything that is need to run backintime.
        also check for hardlink-support on remote host.
        """
        logger.debug('Check remote commands', self)
        def maxArg():
            if retry:
                raise MountException("Checking commands on remote host didn't return any output. "
                                     "We already checked the maximum argument lenght but it seem like "
                                     "there is an other problem")
            logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.',
                           self)
            import sshMaxArg
            mid = sshMaxArg.test_ssh_max_arg(self.user_host)
            sshMaxArg.reportResult(self.host, mid)
            self.config.set_ssh_max_arg_length(mid, self.profile_id)
            return self.check_remote_commands(retry = True)

        #check rsync
        tmp_file = tempfile.mkstemp()[1]
        rsync = tools.get_rsync_prefix( self.config ) + ' --dry-run --chmod=Du+wx %s ' % tmp_file
        rsync += '"%s@%s:%s"' % (self.user, self.host, self.path)
        logger.debug('Check rsync command: %s' %rsync, self)

        #use os.system for compatiblity with snapshots.py
        err = os.system(rsync)
        if err:
            logger.debug('Rsync command returnd error: %s' %err, self)
            os.remove(tmp_file)
            raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
                                    '%(err)s\nLook at \'man backintime\' for further instructions')
                                    % {'host' : self.host, 'command' : rsync, 'err' : err})
        os.remove(tmp_file)

        #check cp chmod find and rm
        remote_tmp_dir = os.path.join(self.path, 'tmp_%s' % self.random_id())
        head  = 'tmp=%s ; ' % remote_tmp_dir
        #first define a function to clean up and exit
        head += 'cleanup(){ '
        head += 'test -e $tmp/a && rm $tmp/a >/dev/null 2>&1; '
        head += 'test -e $tmp/b && rm $tmp/b >/dev/null 2>&1; '
        head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; '
        head += 'test -e $tmp && rmdir $tmp >/dev/null 2>&1; '
        head += 'exit $1; }; '
        tail = []
        #create tmp_RANDOM dir and file a
        cmd  = 'test -e $tmp || mkdir $tmp; touch $tmp/a; '
        tail.append(cmd)

        #try to create hardlink b from a
        cmd  = 'echo \"cp -aRl SOURCE DEST\"; cp -aRl $tmp/a $tmp/b >/dev/null; err_cp=$?; '
        cmd += 'test $err_cp -ne 0 && cleanup $err_cp; '
        tail.append(cmd)
        #list inodes of a and b
        cmd  = 'ls -i $tmp/a; ls -i $tmp/b; '
        tail.append(cmd)
        #try to chmod
        cmd  = 'echo \"chmod u+rw FILE\"; chmod u+rw $tmp/a >/dev/null; err_chmod=$?; '
        cmd += 'test $err_chmod -ne 0 && cleanup $err_chmod; '
        tail.append(cmd)
        #try to find and chmod
        cmd  = 'echo \"find PATH -type f -exec chmod u-wx \"{}\" \\;\"; '
        cmd += 'find $tmp -type f -exec chmod u-wx \"{}\" \\; >/dev/null; err_find=$?; '
        cmd += 'test $err_find -ne 0 && cleanup $err_find; '
        tail.append(cmd)
        #try find suffix '+'
        cmd  = 'find $tmp -type f -exec chmod u-wx \"{}\" + >/dev/null; err_gnu_find=$?; '
        cmd += 'test $err_gnu_find -ne 0 && echo \"gnu_find not supported\"; '
        tail.append(cmd)
        #try to rm -rf
        cmd  = 'echo \"rm -rf PATH\"; rm -rf $tmp >/dev/null; err_rm=$?; '
        cmd += 'test $err_rm -ne 0 && cleanup $err_rm; '
        tail.append(cmd)
        #try nice -n 19
        if self.nice:
            cmd  = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; '
            cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
            tail.append(cmd)
        #try ionice -c2 -n7
        if self.ionice:
            cmd  = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
            cmd += 'test $err_nice -ne 0 && cleanup $err_nice; '
            tail.append(cmd)
        #try nocache
        if self.nocache:
            cmd  = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; '
            cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; '
            tail.append(cmd)
        #try screen, bash and flock used by smart-remove running in background
        if self.config.get_smart_remove_run_remote_in_background(self.profile_id):
            cmd  = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; '
            cmd += 'test $err_screen -ne 0 && cleanup $err_screen; '
            cmd += 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; '
            cmd += 'test $err_flock -ne 0 && cleanup $err_flock; '
            tail.append(cmd)
        #if we end up here, everything should be fine
        cmd = 'echo \"done\"'
        tail.append(cmd)

        maxLength = self.config.ssh_max_arg_length(self.profile_id)
        additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str))

        ssh = ['ssh']
        ssh.extend(self.ssh_options + [self.user_host])
        ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list))
        output = ''
        err = ''
        returncode = 0
        for cmd in tools.splitCommands(tail,
                                       head = head,
                                       maxLength = maxLength,
                                       additionalChars = additionalChars):
            if cmd.endswith('; '):
                cmd += 'echo ""'
            c = ssh[:]
            c.extend([cmd])
            try:
                logger.debug('Call command: %s' %' '.join(c), self)
                proc = subprocess.Popen(c,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        universal_newlines = True)
                ret = proc.communicate()
            except OSError as e:
                #Argument list too long
                if e.errno == 7:
                    logger.debug('Argument list too log (Python exception)', self)
                    return maxArg()
                else:
                    raise
            logger.debug('Command stdout: %s' %ret[0], self)
            logger.debug('Command stderr: %s' %ret[1], self)
            logger.debug('Command returncode: %s' %proc.returncode, self)
            output += ret[0].strip('\n') + '\n'
            err    += ret[1].strip('\n') + '\n'
            returncode += proc.returncode
            if proc.returncode:
                break

        output_split = output.strip('\n').split('\n')

        while True:
            if output_split and not output_split[-1]:
                output_split = output_split[:-1]
            else:
                break

        if not output_split:
            return maxArg()

        gnu_find_suffix_support = True
        for line in output_split:
            if line.startswith('gnu_find not supported'):
                gnu_find_suffix_support = False
        self.config.set_gnu_find_suffix_support(gnu_find_suffix_support, self.profile_id)

        if returncode or not output_split[-1].startswith('done'):
            for command in ('cp', 'chmod', 'find', 'rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'):
                if output_split[-1].startswith(command):
                    raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n'
                                            '%(err)s\nLook at \'man backintime\' for further instructions')
                                            % {'host' : self.host, 'command' : output_split[-1], 'err' : err})
            raise MountException( _('Check commands on host %(host)s returned unknown error:\n'
                                    '%(err)s\nLook at \'man backintime\' for further instructions')
                                    % {'host' : self.host, 'err' : err})

        i = 1
        inode1 = 'ABC'
        inode2 = 'DEF'
        for line in output_split:
            if line.startswith('cp'):
                try:
                    inode1 = output_split[i].split(' ')[0]
                    inode2 = output_split[i+1].split(' ')[0]
                except IndexError:
                    pass
                if not inode1 == inode2:
                    raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
            i += 1
Beispiel #7
0
    def checkRemoteCommands(self, retry=False):
        """
        Try out all relevant commands used by `Back In Time` on the remote host
        to make sure snapshots will be successful with the remote host.
        This will also check that hard-links are supported on the remote host.

        This check can be disabled with :py:func:`config.Config.sshCheckCommands`

        Args:
            retry (bool):               retry to run the commands if it failed
                                        because the command string was to long

        Raises:
            exceptions.MountException:  if a command is not supported on
                                        remote host or if hard-links are not
                                        supported
        """
        if not self.config.sshCheckCommands():
            return
        logger.debug("Check remote commands", self)

        def maxArg():
            if retry:
                raise MountException(
                    "Checking commands on remote host didn't return any output. "
                    "We already checked the maximum argument lenght but it seem like "
                    "there is an other problem"
                )
            logger.warning(
                "Looks like the command was to long for remote SSHd. We will test max arg length now and retry.", self
            )
            import sshMaxArg

            mid = sshMaxArg.maxArgLength(self.config)
            sshMaxArg.reportResult(self.host, mid)
            self.config.setSshMaxArgLength(mid, self.profile_id)
            return self.checkRemoteCommands(retry=True)

        remote_tmp_dir_1 = os.path.join(self.path, "tmp_%s" % self.randomId())
        remote_tmp_dir_2 = os.path.join(self.path, "tmp_%s" % self.randomId())
        with tempfile.TemporaryDirectory() as tmp:
            tmp_file = os.path.join(tmp, "a")
            with open(tmp_file, "wt") as f:
                f.write("foo")

            # check rsync
            rsync1 = tools.rsyncPrefix(self.config, no_perms=False, progress=False)
            rsync1.append(tmp_file)
            rsync1.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address(self.host), remote_tmp_dir_1))

            # check remote rsync hard-link support
            rsync2 = tools.rsyncPrefix(self.config, no_perms=False, progress=False)
            rsync2.append("--link-dest=../%s" % os.path.basename(remote_tmp_dir_1))
            rsync2.append(tmp_file)
            rsync2.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address(self.host), remote_tmp_dir_2))

            for cmd in (rsync1, rsync2):
                logger.debug("Check rsync command: %s" % cmd, self)

                proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
                out, err = proc.communicate()
                if err or proc.returncode:
                    logger.debug("rsync command returned error: %s" % err, self)
                    raise MountException(
                        _(
                            "Remote host %(host)s doesn't support '%(command)s':\n"
                            "%(err)s\nLook at 'man backintime' for further instructions"
                        )
                        % {"host": self.host, "command": cmd, "err": err}
                    )

        # check cp chmod find and rm
        head = 'tmp1="%s"; tmp2="%s"; ' % (remote_tmp_dir_1, remote_tmp_dir_2)
        # first define a function to clean up and exit
        head += "cleanup(){ "
        head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; '
        head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; '
        head += "test -e smr.lock && rm smr.lock >/dev/null 2>&1; "
        head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; '
        head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; '
        head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; '
        head += "exit $1; }; "
        tail = []

        # list inodes
        cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; '
        tail.append(cmd)
        # try nice -n 19
        if self.nice:
            cmd = 'echo "nice -n 19"; nice -n 19 true >/dev/null; err_nice=$?; '
            cmd += "test $err_nice -ne 0 && cleanup $err_nice; "
            tail.append(cmd)
        # try ionice -c2 -n7
        if self.ionice:
            cmd = 'echo "ionice -c2 -n7"; ionice -c2 -n7 true >/dev/null; err_nice=$?; '
            cmd += "test $err_nice -ne 0 && cleanup $err_nice; "
            tail.append(cmd)
        # try nocache
        if self.nocache:
            cmd = 'echo "nocache"; nocache true >/dev/null; err_nocache=$?; '
            cmd += "test $err_nocache -ne 0 && cleanup $err_nocache; "
            tail.append(cmd)
        # try screen, bash and flock used by smart-remove running in background
        if self.config.smartRemoveRunRemoteInBackground(self.profile_id):
            cmd = 'echo "screen -d -m bash -c ..."; screen -d -m bash -c "true" >/dev/null; err_screen=$?; '
            cmd += "test $err_screen -ne 0 && cleanup $err_screen; "
            tail.append(cmd)
            cmd = 'echo "(flock -x 9) 9>smr.lock"; bash -c "(flock -x 9) 9>smr.lock" >/dev/null; err_flock=$?; '
            cmd += "test $err_flock -ne 0 && cleanup $err_flock; "
            tail.append(cmd)
            cmd = 'echo "rmdir \$(mktemp -d)"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; '
            cmd += "test $err_rmdir -ne 0 && cleanup $err_rmdir; "
            tail.append(cmd)
        # if we end up here, everything should be fine
        cmd = 'echo "done"; cleanup 0'
        tail.append(cmd)

        maxLength = self.config.sshMaxArgLength(self.profile_id)
        additionalChars = len('echo ""') + len(self.config.sshPrefixCmd(self.profile_id, cmd_type=str))

        output = ""
        err = ""
        returncode = 0
        for cmd in tools.splitCommands(tail, head=head, maxLength=maxLength - additionalChars):
            if cmd.endswith("; "):
                cmd += 'echo ""'
            c = self.config.sshCommand(
                cmd=[cmd],
                custom_args=["-p", str(self.port), self.user_host],
                port=False,
                cipher=False,
                user_host=False,
                nice=False,
                ionice=False,
                profile_id=self.profile_id,
            )
            try:
                logger.debug("Call command: %s" % " ".join(c), self)
                proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
                ret = proc.communicate()
            except OSError as e:
                # Argument list too long
                if e.errno == 7:
                    logger.debug("Argument list too log (Python exception)", self)
                    return maxArg()
                else:
                    raise
            logger.debug("Command stdout: %s" % ret[0], self)
            logger.debug("Command stderr: %s" % ret[1], self)
            logger.debug("Command returncode: %s" % proc.returncode, self)
            output += ret[0].strip("\n") + "\n"
            err += ret[1].strip("\n") + "\n"
            returncode += proc.returncode
            if proc.returncode:
                break

        output_split = output.strip("\n").split("\n")

        while True:
            if output_split and not output_split[-1]:
                output_split = output_split[:-1]
            else:
                break

        if not output_split:
            return maxArg()

        if returncode or not output_split[-1].startswith("done"):
            for command in ("rm", "nice", "ionice", "nocache", "screen", "(flock"):
                if output_split[-1].startswith(command):
                    raise MountException(
                        _(
                            "Remote host %(host)s doesn't support '%(command)s':\n"
                            "%(err)s\nLook at 'man backintime' for further instructions"
                        )
                        % {"host": self.host, "command": output_split[-1], "err": err}
                    )
            raise MountException(
                _(
                    "Check commands on host %(host)s returned unknown error:\n"
                    "%(err)s\nLook at 'man backintime' for further instructions"
                )
                % {"host": self.host, "err": err}
            )

        inodes = []
        for tmp in (remote_tmp_dir_1, remote_tmp_dir_2):
            for line in output_split:
                m = re.match(r"^(\d+).*?%s" % tmp, line)
                if m:
                    inodes.append(m.group(1))

        logger.debug("remote inodes: " + " | ".join(inodes), self)
        if len(inodes) == 2 and inodes[0] != inodes[1]:
            raise MountException(_("Remote host %s doesn't support hardlinks") % self.host)