def test_splitCommands(self): ret = list( tools.splitCommands(['echo foo;'], head='echo start;', tail='echo end', maxLength=40)) self.assertEqual(len(ret), 1) self.assertEqual(ret[0], 'echo start;echo foo;echo end') ret = list( tools.splitCommands(['echo foo;'] * 3, head='echo start;', tail='echo end', maxLength=40)) self.assertEqual(len(ret), 2) self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo end') self.assertEqual(ret[1], 'echo start;echo foo;echo end') ret = list( tools.splitCommands(['echo foo;'] * 3, head='echo start;', tail='echo end', maxLength=0)) self.assertEqual(len(ret), 1) self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo foo;echo end') ret = list( tools.splitCommands(['echo foo;'] * 3, head='echo start;', tail='echo end', maxLength=-10)) self.assertEqual(len(ret), 1) self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo foo;echo end')
def test_splitCommands(self): ret = list(tools.splitCommands(['echo foo;'], head = 'echo start;', tail = 'echo end', maxLength = 40)) self.assertEqual(len(ret), 1) self.assertEqual(ret[0], 'echo start;echo foo;echo end') ret = list(tools.splitCommands(['echo foo;']*3, head = 'echo start;', tail = 'echo end', maxLength = 40)) self.assertEqual(len(ret), 2) self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo end') self.assertEqual(ret[1], 'echo start;echo foo;echo end') ret = list(tools.splitCommands(['echo foo;']*3, head = 'echo start;', tail = 'echo end', maxLength = 0)) self.assertEqual(len(ret), 1) self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo foo;echo end') ret = list(tools.splitCommands(['echo foo;']*3, head = 'echo start;', tail = 'echo end', maxLength = -10)) self.assertEqual(len(ret), 1) self.assertEqual(ret[0], 'echo start;echo foo;echo foo;echo foo;echo end')
def check_remote_commands(self, retry = False): """ try all relevant commands for take_snapshot on remote host. specialy embedded Linux devices using 'BusyBox' sometimes doesn't support everything that is need to run backintime. also check for hardlink-support on remote host. """ logger.debug('Check remote commands', self) def maxArg(): if retry: raise MountException("Checking commands on remote host didn't return any output. " "We already checked the maximum argument lenght but it seem like " "there is an other problem") logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.', self) import sshMaxArg mid = sshMaxArg.test_ssh_max_arg(self.user_host) sshMaxArg.reportResult(self.host, mid) self.config.set_ssh_max_arg_length(mid, self.profile_id) return self.check_remote_commands(retry = True) #check rsync tmp_file = tempfile.mkstemp()[1] rsync = tools.get_rsync_prefix( self.config ) + ' --dry-run --chmod=Du+wx %s ' % tmp_file rsync += '"%s@%s:%s"' % (self.user, self.host, self.path) logger.debug('Check rsync command: %s' %rsync, self) #use os.system for compatiblity with snapshots.py err = os.system(rsync) if err: logger.debug('Rsync command returnd error: %s' %err, self) os.remove(tmp_file) raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'command' : rsync, 'err' : err}) os.remove(tmp_file) #check cp chmod find and rm remote_tmp_dir = os.path.join(self.path, 'tmp_%s' % self.random_id()) head = 'tmp=%s ; ' % remote_tmp_dir #first define a function to clean up and exit head += 'cleanup(){ ' head += 'test -e $tmp/a && rm $tmp/a >/dev/null 2>&1; ' head += 'test -e $tmp/b && rm $tmp/b >/dev/null 2>&1; ' head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; ' head += 'test -e $tmp && rmdir $tmp >/dev/null 2>&1; ' head += 'exit $1; }; ' tail = [] #create tmp_RANDOM dir and file a cmd = 'test -e $tmp || mkdir $tmp; touch $tmp/a; ' tail.append(cmd) #try to create hardlink b from a cmd = 'echo \"cp -aRl SOURCE DEST\"; cp -aRl $tmp/a $tmp/b >/dev/null; err_cp=$?; ' cmd += 'test $err_cp -ne 0 && cleanup $err_cp; ' tail.append(cmd) #list inodes of a and b cmd = 'ls -i $tmp/a; ls -i $tmp/b; ' tail.append(cmd) #try to chmod cmd = 'echo \"chmod u+rw FILE\"; chmod u+rw $tmp/a >/dev/null; err_chmod=$?; ' cmd += 'test $err_chmod -ne 0 && cleanup $err_chmod; ' tail.append(cmd) #try to find and chmod cmd = 'echo \"find PATH -type f -exec chmod u-wx \"{}\" \\;\"; ' cmd += 'find $tmp -type f -exec chmod u-wx \"{}\" \\; >/dev/null; err_find=$?; ' cmd += 'test $err_find -ne 0 && cleanup $err_find; ' tail.append(cmd) #try find suffix '+' cmd = 'find $tmp -type f -exec chmod u-wx \"{}\" + >/dev/null; err_gnu_find=$?; ' cmd += 'test $err_gnu_find -ne 0 && echo \"gnu_find not supported\"; ' tail.append(cmd) #try to rm -rf cmd = 'echo \"rm -rf PATH\"; rm -rf $tmp >/dev/null; err_rm=$?; ' cmd += 'test $err_rm -ne 0 && cleanup $err_rm; ' tail.append(cmd) #try nice -n 19 if self.nice: cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try ionice -c2 -n7 if self.ionice: cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try nocache if self.nocache: cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; ' cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; ' tail.append(cmd) #try screen, bash and flock used by smart-remove running in background if self.config.get_smart_remove_run_remote_in_background(self.profile_id): cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; ' cmd += 'test $err_screen -ne 0 && cleanup $err_screen; ' cmd += 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; ' cmd += 'test $err_flock -ne 0 && cleanup $err_flock; ' tail.append(cmd) #if we end up here, everything should be fine cmd = 'echo \"done\"' tail.append(cmd) maxLength = self.config.ssh_max_arg_length(self.profile_id) additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str)) ssh = ['ssh'] ssh.extend(self.ssh_options + [self.user_host]) ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list)) output = '' err = '' returncode = 0 for cmd in tools.splitCommands(tail, head = head, maxLength = maxLength - additionalChars): if cmd.endswith('; '): cmd += 'echo ""' c = ssh[:] c.extend([cmd]) try: logger.debug('Call command: %s' %' '.join(c), self) proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines = True) ret = proc.communicate() except OSError as e: #Argument list too long if e.errno == 7: logger.debug('Argument list too log (Python exception)', self) return maxArg() else: raise logger.debug('Command stdout: %s' %ret[0], self) logger.debug('Command stderr: %s' %ret[1], self) logger.debug('Command returncode: %s' %proc.returncode, self) output += ret[0].strip('\n') + '\n' err += ret[1].strip('\n') + '\n' returncode += proc.returncode if proc.returncode: break output_split = output.strip('\n').split('\n') while True: if output_split and not output_split[-1]: output_split = output_split[:-1] else: break if not output_split: return maxArg() gnu_find_suffix_support = True for line in output_split: if line.startswith('gnu_find not supported'): gnu_find_suffix_support = False self.config.set_gnu_find_suffix_support(gnu_find_suffix_support, self.profile_id) if returncode or not output_split[-1].startswith('done'): for command in ('cp', 'chmod', 'find', 'rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'): if output_split[-1].startswith(command): raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'command' : output_split[-1], 'err' : err}) raise MountException( _('Check commands on host %(host)s returned unknown error:\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'err' : err}) i = 1 inode1 = 'ABC' inode2 = 'DEF' for line in output_split: if line.startswith('cp'): try: inode1 = output_split[i].split(' ')[0] inode2 = output_split[i+1].split(' ')[0] except IndexError: pass if not inode1 == inode2: raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host) i += 1
def checkRemoteCommands(self, retry=False): """ Try out all relevant commands used by `Back In Time` on the remote host to make sure snapshots will be successful with the remote host. This will also check that hard-links are supported on the remote host. This check can be disabled with :py:func:`config.Config.sshCheckCommands` Args: retry (bool): retry to run the commands if it failed because the command string was to long Raises: exceptions.MountException: if a command is not supported on remote host or if hard-links are not supported """ if not self.config.sshCheckCommands(): return logger.debug('Check remote commands', self) def maxArg(): if retry: raise MountException( "Checking commands on remote host didn't return any output. " "We already checked the maximum argument lenght but it seem like " "there is an other problem") logger.warning( 'Looks like the command was to long for remote SSHd. We will test max arg length now and retry.', self) import sshMaxArg mid = sshMaxArg.maxArgLength(self.config) sshMaxArg.reportResult(self.host, mid) self.config.setSshMaxArgLength(mid, self.profile_id) return self.checkRemoteCommands(retry=True) remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.randomId()) remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.randomId()) with tempfile.TemporaryDirectory() as tmp: tmp_file = os.path.join(tmp, 'a') with open(tmp_file, 'wt') as f: f.write('foo') #check rsync rsync1 = tools.rsyncPrefix(self.config, no_perms=False, progress=False) rsync1.append(tmp_file) rsync1.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address( self.host), remote_tmp_dir_1)) #check remote rsync hard-link support rsync2 = tools.rsyncPrefix(self.config, no_perms=False, progress=False) rsync2.append('--link-dest=../%s' % os.path.basename(remote_tmp_dir_1)) rsync2.append(tmp_file) rsync2.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address( self.host), remote_tmp_dir_2)) for cmd in (rsync1, rsync2): logger.debug('Check rsync command: %s' % cmd, self) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = proc.communicate() if err or proc.returncode: logger.debug('rsync command returned error: %s' % err, self) raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions' ) % { 'host': self.host, 'command': cmd, 'err': err }) #check cp chmod find and rm head = 'tmp1="%s"; tmp2="%s"; ' % (remote_tmp_dir_1, remote_tmp_dir_2) #first define a function to clean up and exit head += 'cleanup(){ ' head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; ' head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; ' head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; ' head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; ' head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; ' head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; ' head += 'exit $1; }; ' tail = [] #list inodes cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; ' tail.append(cmd) #try nice -n 19 if self.nice: cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try ionice -c2 -n7 if self.ionice: cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try nocache if self.nocache: cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; ' cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; ' tail.append(cmd) #try screen, bash and flock used by smart-remove running in background if self.config.smartRemoveRunRemoteInBackground(self.profile_id): cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; ' cmd += 'test $err_screen -ne 0 && cleanup $err_screen; ' tail.append(cmd) cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; ' cmd += 'test $err_flock -ne 0 && cleanup $err_flock; ' tail.append(cmd) cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; ' cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; ' tail.append(cmd) #if we end up here, everything should be fine cmd = 'echo \"done\"; cleanup 0' tail.append(cmd) maxLength = self.config.sshMaxArgLength(self.profile_id) additionalChars = len('echo ""') + len( self.config.sshPrefixCmd(self.profile_id, cmd_type=str)) output = '' err = '' returncode = 0 for cmd in tools.splitCommands(tail, head=head, maxLength=maxLength - additionalChars): if cmd.endswith('; '): cmd += 'echo ""' c = self.config.sshCommand( cmd=[cmd], custom_args=['-p', str(self.port), self.user_host], port=False, user_host=False, nice=False, ionice=False, profile_id=self.profile_id) try: logger.debug('Call command: %s' % ' '.join(c), self) proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) ret = proc.communicate() except OSError as e: #Argument list too long if e.errno == 7: logger.debug('Argument list too log (Python exception)', self) return maxArg() else: raise logger.debug('Command stdout: %s' % ret[0], self) logger.debug('Command stderr: %s' % ret[1], self) logger.debug('Command returncode: %s' % proc.returncode, self) output += ret[0].strip('\n') + '\n' err += ret[1].strip('\n') + '\n' returncode += proc.returncode if proc.returncode: break output_split = output.strip('\n').split('\n') while True: if output_split and not output_split[-1]: output_split = output_split[:-1] else: break if not output_split: return maxArg() if returncode or not output_split[-1].startswith('done'): for command in ('rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'): if output_split[-1].startswith(command): raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions' ) % { 'host': self.host, 'command': output_split[-1], 'err': err }) raise MountException( _('Check commands on host %(host)s returned unknown error:\n' '%(err)s\nLook at \'man backintime\' for further instructions' ) % { 'host': self.host, 'err': err }) inodes = [] for tmp in (remote_tmp_dir_1, remote_tmp_dir_2): for line in output_split: m = re.match(r'^(\d+).*?%s' % tmp, line) if m: inodes.append(m.group(1)) logger.debug('remote inodes: ' + ' | '.join(inodes), self) if len(inodes) == 2 and inodes[0] != inodes[1]: raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
def check_remote_commands(self, retry = False): """ Try out all relevant commands used by `Back In Time` on the remote host to make sure snapshots will be successful with the remote host. This will also check that hard-links are supported on the remote host. This check can be disabled with :py:func:`config.Config.ssh_check_commands` Args: retry (bool): retry to run the commands if it failed because the command string was to long Raises: exceptions.MountException: if a command is not supported on remote host or if hard-links are not supported """ if not self.config.ssh_check_commands(): return logger.debug('Check remote commands', self) def maxArg(): if retry: raise MountException("Checking commands on remote host didn't return any output. " "We already checked the maximum argument lenght but it seem like " "there is an other problem") logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.', self) import sshMaxArg mid = sshMaxArg.test_ssh_max_arg(self.user_host) sshMaxArg.reportResult(self.host, mid) self.config.set_ssh_max_arg_length(mid, self.profile_id) return self.check_remote_commands(retry = True) remote_tmp_dir_1 = os.path.join(self.path, 'tmp_%s' % self.random_id()) remote_tmp_dir_2 = os.path.join(self.path, 'tmp_%s' % self.random_id()) with tempfile.TemporaryDirectory() as tmp: tmp_file = os.path.join(tmp, 'a') with open(tmp_file, 'wt') as f: f.write('foo') #check rsync rsync1 = tools.get_rsync_prefix(self.config, no_perms = False, progress = False) rsync1.append(tmp_file) rsync1.append('%s@%s:%s/' %(self.user, self.host, remote_tmp_dir_1)) #check remote rsync hard-link support rsync2 = tools.get_rsync_prefix(self.config, no_perms = False, progress = False) rsync2.append('--link-dest=../%s' %os.path.basename(remote_tmp_dir_1)) rsync2.append(tmp_file) rsync2.append('%s@%s:%s/' %(self.user, self.host, remote_tmp_dir_2)) for cmd in (rsync1, rsync2): logger.debug('Check rsync command: %s' %cmd, self) proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, universal_newlines = True) out, err = proc.communicate() if err or proc.returncode: logger.debug('rsync command returned error: %s' %err, self) raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'command' : cmd, 'err' : err}) #check cp chmod find and rm head = 'tmp1=%s; tmp2=%s; ' %(remote_tmp_dir_1, remote_tmp_dir_2) #first define a function to clean up and exit head += 'cleanup(){ ' head += 'test -e $tmp1/a && rm $tmp1/a >/dev/null 2>&1; ' head += 'test -e $tmp2/a && rm $tmp2/a >/dev/null 2>&1; ' head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; ' head += 'test -e $tmp1 && rmdir $tmp1 >/dev/null 2>&1; ' head += 'test -e $tmp2 && rmdir $tmp2 >/dev/null 2>&1; ' head += 'test -n "$tmp3" && test -e $tmp3 && rmdir $tmp3 >/dev/null 2>&1; ' head += 'exit $1; }; ' tail = [] #list inodes cmd = 'ls -i $tmp1/a; ls -i $tmp2/a; ' tail.append(cmd) #try nice -n 19 if self.nice: cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try ionice -c2 -n7 if self.ionice: cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try nocache if self.nocache: cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; ' cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; ' tail.append(cmd) #try screen, bash and flock used by smart-remove running in background if self.config.get_smart_remove_run_remote_in_background(self.profile_id): cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; ' cmd += 'test $err_screen -ne 0 && cleanup $err_screen; ' tail.append(cmd) cmd = 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; ' cmd += 'test $err_flock -ne 0 && cleanup $err_flock; ' tail.append(cmd) cmd = 'echo \"rmdir \$(mktemp -d)\"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; ' cmd += 'test $err_rmdir -ne 0 && cleanup $err_rmdir; ' tail.append(cmd) #if we end up here, everything should be fine cmd = 'echo \"done\"; cleanup 0' tail.append(cmd) maxLength = self.config.ssh_max_arg_length(self.profile_id) additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str)) ssh = ['ssh'] ssh.extend(self.ssh_options + [self.user_host]) ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list)) output = '' err = '' returncode = 0 for cmd in tools.splitCommands(tail, head = head, maxLength = maxLength - additionalChars): if cmd.endswith('; '): cmd += 'echo ""' c = ssh[:] c.extend([cmd]) try: logger.debug('Call command: %s' %' '.join(c), self) proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines = True) ret = proc.communicate() except OSError as e: #Argument list too long if e.errno == 7: logger.debug('Argument list too log (Python exception)', self) return maxArg() else: raise logger.debug('Command stdout: %s' %ret[0], self) logger.debug('Command stderr: %s' %ret[1], self) logger.debug('Command returncode: %s' %proc.returncode, self) output += ret[0].strip('\n') + '\n' err += ret[1].strip('\n') + '\n' returncode += proc.returncode if proc.returncode: break output_split = output.strip('\n').split('\n') while True: if output_split and not output_split[-1]: output_split = output_split[:-1] else: break if not output_split: return maxArg() if returncode or not output_split[-1].startswith('done'): for command in ('rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'): if output_split[-1].startswith(command): raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'command' : output_split[-1], 'err' : err}) raise MountException( _('Check commands on host %(host)s returned unknown error:\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'err' : err}) inodes = [] for tmp in (remote_tmp_dir_1, remote_tmp_dir_2): for line in output_split: m = re.match(r'^(\d+).*?%s' %tmp, line) if m: inodes.append(m.group(1)) logger.debug('remote inodes: ' + ' | '.join(inodes), self) if len(inodes) == 2 and inodes[0] != inodes[1]: raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host)
def check_remote_commands(self, retry = False): """ try all relevant commands for take_snapshot on remote host. specialy embedded Linux devices using 'BusyBox' sometimes doesn't support everything that is need to run backintime. also check for hardlink-support on remote host. """ logger.debug('Check remote commands', self) def maxArg(): if retry: raise MountException("Checking commands on remote host didn't return any output. " "We already checked the maximum argument lenght but it seem like " "there is an other problem") logger.warning('Looks like the command was to long for remote SSHd. We will test max arg length now and retry.', self) import sshMaxArg mid = sshMaxArg.test_ssh_max_arg(self.user_host) sshMaxArg.reportResult(self.host, mid) self.config.set_ssh_max_arg_length(mid, self.profile_id) return self.check_remote_commands(retry = True) #check rsync tmp_file = tempfile.mkstemp()[1] rsync = tools.get_rsync_prefix( self.config ) + ' --dry-run --chmod=Du+wx %s ' % tmp_file rsync += '"%s@%s:%s"' % (self.user, self.host, self.path) logger.debug('Check rsync command: %s' %rsync, self) #use os.system for compatiblity with snapshots.py err = os.system(rsync) if err: logger.debug('Rsync command returnd error: %s' %err, self) os.remove(tmp_file) raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'command' : rsync, 'err' : err}) os.remove(tmp_file) #check cp chmod find and rm remote_tmp_dir = os.path.join(self.path, 'tmp_%s' % self.random_id()) head = 'tmp=%s ; ' % remote_tmp_dir #first define a function to clean up and exit head += 'cleanup(){ ' head += 'test -e $tmp/a && rm $tmp/a >/dev/null 2>&1; ' head += 'test -e $tmp/b && rm $tmp/b >/dev/null 2>&1; ' head += 'test -e smr.lock && rm smr.lock >/dev/null 2>&1; ' head += 'test -e $tmp && rmdir $tmp >/dev/null 2>&1; ' head += 'exit $1; }; ' tail = [] #create tmp_RANDOM dir and file a cmd = 'test -e $tmp || mkdir $tmp; touch $tmp/a; ' tail.append(cmd) #try to create hardlink b from a cmd = 'echo \"cp -aRl SOURCE DEST\"; cp -aRl $tmp/a $tmp/b >/dev/null; err_cp=$?; ' cmd += 'test $err_cp -ne 0 && cleanup $err_cp; ' tail.append(cmd) #list inodes of a and b cmd = 'ls -i $tmp/a; ls -i $tmp/b; ' tail.append(cmd) #try to chmod cmd = 'echo \"chmod u+rw FILE\"; chmod u+rw $tmp/a >/dev/null; err_chmod=$?; ' cmd += 'test $err_chmod -ne 0 && cleanup $err_chmod; ' tail.append(cmd) #try to find and chmod cmd = 'echo \"find PATH -type f -exec chmod u-wx \"{}\" \\;\"; ' cmd += 'find $tmp -type f -exec chmod u-wx \"{}\" \\; >/dev/null; err_find=$?; ' cmd += 'test $err_find -ne 0 && cleanup $err_find; ' tail.append(cmd) #try find suffix '+' cmd = 'find $tmp -type f -exec chmod u-wx \"{}\" + >/dev/null; err_gnu_find=$?; ' cmd += 'test $err_gnu_find -ne 0 && echo \"gnu_find not supported\"; ' tail.append(cmd) #try to rm -rf cmd = 'echo \"rm -rf PATH\"; rm -rf $tmp >/dev/null; err_rm=$?; ' cmd += 'test $err_rm -ne 0 && cleanup $err_rm; ' tail.append(cmd) #try nice -n 19 if self.nice: cmd = 'echo \"nice -n 19\"; nice -n 19 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try ionice -c2 -n7 if self.ionice: cmd = 'echo \"ionice -c2 -n7\"; ionice -c2 -n7 true >/dev/null; err_nice=$?; ' cmd += 'test $err_nice -ne 0 && cleanup $err_nice; ' tail.append(cmd) #try nocache if self.nocache: cmd = 'echo \"nocache\"; nocache true >/dev/null; err_nocache=$?; ' cmd += 'test $err_nocache -ne 0 && cleanup $err_nocache; ' tail.append(cmd) #try screen, bash and flock used by smart-remove running in background if self.config.get_smart_remove_run_remote_in_background(self.profile_id): cmd = 'echo \"screen -d -m bash -c ...\"; screen -d -m bash -c \"true\" >/dev/null; err_screen=$?; ' cmd += 'test $err_screen -ne 0 && cleanup $err_screen; ' cmd += 'echo \"(flock -x 9) 9>smr.lock\"; bash -c \"(flock -x 9) 9>smr.lock\" >/dev/null; err_flock=$?; ' cmd += 'test $err_flock -ne 0 && cleanup $err_flock; ' tail.append(cmd) #if we end up here, everything should be fine cmd = 'echo \"done\"' tail.append(cmd) maxLength = self.config.ssh_max_arg_length(self.profile_id) additionalChars = len('echo ""') + len(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = str)) ssh = ['ssh'] ssh.extend(self.ssh_options + [self.user_host]) ssh.extend(self.config.ssh_prefix_cmd(self.profile_id, cmd_type = list)) output = '' err = '' returncode = 0 for cmd in tools.splitCommands(tail, head = head, maxLength = maxLength, additionalChars = additionalChars): if cmd.endswith('; '): cmd += 'echo ""' c = ssh[:] c.extend([cmd]) try: logger.debug('Call command: %s' %' '.join(c), self) proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines = True) ret = proc.communicate() except OSError as e: #Argument list too long if e.errno == 7: logger.debug('Argument list too log (Python exception)', self) return maxArg() else: raise logger.debug('Command stdout: %s' %ret[0], self) logger.debug('Command stderr: %s' %ret[1], self) logger.debug('Command returncode: %s' %proc.returncode, self) output += ret[0].strip('\n') + '\n' err += ret[1].strip('\n') + '\n' returncode += proc.returncode if proc.returncode: break output_split = output.strip('\n').split('\n') while True: if output_split and not output_split[-1]: output_split = output_split[:-1] else: break if not output_split: return maxArg() gnu_find_suffix_support = True for line in output_split: if line.startswith('gnu_find not supported'): gnu_find_suffix_support = False self.config.set_gnu_find_suffix_support(gnu_find_suffix_support, self.profile_id) if returncode or not output_split[-1].startswith('done'): for command in ('cp', 'chmod', 'find', 'rm', 'nice', 'ionice', 'nocache', 'screen', '(flock'): if output_split[-1].startswith(command): raise MountException( _('Remote host %(host)s doesn\'t support \'%(command)s\':\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'command' : output_split[-1], 'err' : err}) raise MountException( _('Check commands on host %(host)s returned unknown error:\n' '%(err)s\nLook at \'man backintime\' for further instructions') % {'host' : self.host, 'err' : err}) i = 1 inode1 = 'ABC' inode2 = 'DEF' for line in output_split: if line.startswith('cp'): try: inode1 = output_split[i].split(' ')[0] inode2 = output_split[i+1].split(' ')[0] except IndexError: pass if not inode1 == inode2: raise MountException( _('Remote host %s doesn\'t support hardlinks') % self.host) i += 1
def checkRemoteCommands(self, retry=False): """ Try out all relevant commands used by `Back In Time` on the remote host to make sure snapshots will be successful with the remote host. This will also check that hard-links are supported on the remote host. This check can be disabled with :py:func:`config.Config.sshCheckCommands` Args: retry (bool): retry to run the commands if it failed because the command string was to long Raises: exceptions.MountException: if a command is not supported on remote host or if hard-links are not supported """ if not self.config.sshCheckCommands(): return logger.debug("Check remote commands", self) def maxArg(): if retry: raise MountException( "Checking commands on remote host didn't return any output. " "We already checked the maximum argument lenght but it seem like " "there is an other problem" ) logger.warning( "Looks like the command was to long for remote SSHd. We will test max arg length now and retry.", self ) import sshMaxArg mid = sshMaxArg.maxArgLength(self.config) sshMaxArg.reportResult(self.host, mid) self.config.setSshMaxArgLength(mid, self.profile_id) return self.checkRemoteCommands(retry=True) remote_tmp_dir_1 = os.path.join(self.path, "tmp_%s" % self.randomId()) remote_tmp_dir_2 = os.path.join(self.path, "tmp_%s" % self.randomId()) with tempfile.TemporaryDirectory() as tmp: tmp_file = os.path.join(tmp, "a") with open(tmp_file, "wt") as f: f.write("foo") # check rsync rsync1 = tools.rsyncPrefix(self.config, no_perms=False, progress=False) rsync1.append(tmp_file) rsync1.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address(self.host), remote_tmp_dir_1)) # check remote rsync hard-link support rsync2 = tools.rsyncPrefix(self.config, no_perms=False, progress=False) rsync2.append("--link-dest=../%s" % os.path.basename(remote_tmp_dir_1)) rsync2.append(tmp_file) rsync2.append('%s@%s:"%s"/' % (self.user, tools.escapeIPv6Address(self.host), remote_tmp_dir_2)) for cmd in (rsync1, rsync2): logger.debug("Check rsync command: %s" % cmd, self) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = proc.communicate() if err or proc.returncode: logger.debug("rsync command returned error: %s" % err, self) raise MountException( _( "Remote host %(host)s doesn't support '%(command)s':\n" "%(err)s\nLook at 'man backintime' for further instructions" ) % {"host": self.host, "command": cmd, "err": err} ) # check cp chmod find and rm head = 'tmp1="%s"; tmp2="%s"; ' % (remote_tmp_dir_1, remote_tmp_dir_2) # first define a function to clean up and exit head += "cleanup(){ " head += 'test -e "$tmp1/a" && rm "$tmp1/a" >/dev/null 2>&1; ' head += 'test -e "$tmp2/a" && rm "$tmp2/a" >/dev/null 2>&1; ' head += "test -e smr.lock && rm smr.lock >/dev/null 2>&1; " head += 'test -e "$tmp1" && rmdir "$tmp1" >/dev/null 2>&1; ' head += 'test -e "$tmp2" && rmdir "$tmp2" >/dev/null 2>&1; ' head += 'test -n "$tmp3" && test -e "$tmp3" && rmdir "$tmp3" >/dev/null 2>&1; ' head += "exit $1; }; " tail = [] # list inodes cmd = 'ls -i "$tmp1/a"; ls -i "$tmp2/a"; ' tail.append(cmd) # try nice -n 19 if self.nice: cmd = 'echo "nice -n 19"; nice -n 19 true >/dev/null; err_nice=$?; ' cmd += "test $err_nice -ne 0 && cleanup $err_nice; " tail.append(cmd) # try ionice -c2 -n7 if self.ionice: cmd = 'echo "ionice -c2 -n7"; ionice -c2 -n7 true >/dev/null; err_nice=$?; ' cmd += "test $err_nice -ne 0 && cleanup $err_nice; " tail.append(cmd) # try nocache if self.nocache: cmd = 'echo "nocache"; nocache true >/dev/null; err_nocache=$?; ' cmd += "test $err_nocache -ne 0 && cleanup $err_nocache; " tail.append(cmd) # try screen, bash and flock used by smart-remove running in background if self.config.smartRemoveRunRemoteInBackground(self.profile_id): cmd = 'echo "screen -d -m bash -c ..."; screen -d -m bash -c "true" >/dev/null; err_screen=$?; ' cmd += "test $err_screen -ne 0 && cleanup $err_screen; " tail.append(cmd) cmd = 'echo "(flock -x 9) 9>smr.lock"; bash -c "(flock -x 9) 9>smr.lock" >/dev/null; err_flock=$?; ' cmd += "test $err_flock -ne 0 && cleanup $err_flock; " tail.append(cmd) cmd = 'echo "rmdir \$(mktemp -d)"; tmp3=$(mktemp -d); test -z "$tmp3" && cleanup 1; rmdir $tmp3 >/dev/null; err_rmdir=$?; ' cmd += "test $err_rmdir -ne 0 && cleanup $err_rmdir; " tail.append(cmd) # if we end up here, everything should be fine cmd = 'echo "done"; cleanup 0' tail.append(cmd) maxLength = self.config.sshMaxArgLength(self.profile_id) additionalChars = len('echo ""') + len(self.config.sshPrefixCmd(self.profile_id, cmd_type=str)) output = "" err = "" returncode = 0 for cmd in tools.splitCommands(tail, head=head, maxLength=maxLength - additionalChars): if cmd.endswith("; "): cmd += 'echo ""' c = self.config.sshCommand( cmd=[cmd], custom_args=["-p", str(self.port), self.user_host], port=False, cipher=False, user_host=False, nice=False, ionice=False, profile_id=self.profile_id, ) try: logger.debug("Call command: %s" % " ".join(c), self) proc = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) ret = proc.communicate() except OSError as e: # Argument list too long if e.errno == 7: logger.debug("Argument list too log (Python exception)", self) return maxArg() else: raise logger.debug("Command stdout: %s" % ret[0], self) logger.debug("Command stderr: %s" % ret[1], self) logger.debug("Command returncode: %s" % proc.returncode, self) output += ret[0].strip("\n") + "\n" err += ret[1].strip("\n") + "\n" returncode += proc.returncode if proc.returncode: break output_split = output.strip("\n").split("\n") while True: if output_split and not output_split[-1]: output_split = output_split[:-1] else: break if not output_split: return maxArg() if returncode or not output_split[-1].startswith("done"): for command in ("rm", "nice", "ionice", "nocache", "screen", "(flock"): if output_split[-1].startswith(command): raise MountException( _( "Remote host %(host)s doesn't support '%(command)s':\n" "%(err)s\nLook at 'man backintime' for further instructions" ) % {"host": self.host, "command": output_split[-1], "err": err} ) raise MountException( _( "Check commands on host %(host)s returned unknown error:\n" "%(err)s\nLook at 'man backintime' for further instructions" ) % {"host": self.host, "err": err} ) inodes = [] for tmp in (remote_tmp_dir_1, remote_tmp_dir_2): for line in output_split: m = re.match(r"^(\d+).*?%s" % tmp, line) if m: inodes.append(m.group(1)) logger.debug("remote inodes: " + " | ".join(inodes), self) if len(inodes) == 2 and inodes[0] != inodes[1]: raise MountException(_("Remote host %s doesn't support hardlinks") % self.host)