def test_list_files(self): """ Unit test for RsyncCopyController._list_file's code """ # Mock rsync invocation rsync_mock = mock.Mock(name='Rsync()') rsync_mock.ret = 0 rsync_mock.out = 'drwxrwxrwt 69632 2015/02/09 15:01:00 tmp\n' \ 'drwxrwxrwt 69612 Thu Feb 19 15:01:22 2015 tmp2' rsync_mock.err = 'err' # Test the _list_files internal method rcc = RsyncCopyController() return_values = list(rcc._list_files(rsync_mock, 'some/path')) # Returned list must contain two elements assert len(return_values) == 2 # Check rsync.get_output has called correctly rsync_mock.get_output.assert_called_with('--no-human-readable', '--list-only', '-r', 'some/path', check=True) # Check the result assert return_values[0] == _FileItem( 'drwxrwxrwt', 69632, datetime(year=2015, month=2, day=9, hour=15, minute=1, second=0, tzinfo=dateutil.tz.tzlocal()), 'tmp') assert return_values[1] == _FileItem( 'drwxrwxrwt', 69612, datetime(year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal()), 'tmp2') # Test the _list_files internal method with a wrong output (added TZ) rsync_mock.out = ( 'drwxrwxrwt 69612 Thu Feb 19 15:01:22 CET 2015 tmp2\n') rcc = RsyncCopyController() with pytest.raises(RsyncListFilesFailure): # The list() call is needed to consume the generator list(rcc._list_files(rsync_mock, 'some/path')) # Check rsync.get_output has called correctly rsync_mock.get_output.assert_called_with('--no-human-readable', '--list-only', '-r', 'some/path', check=True)
def test_fill_buckets(self): """ Unit test for RsyncCopyController._fill_buckets's code """ # Create a fake file list af about 525 GB of files filedate = datetime( year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal(), ) file_list = [] total_size = 0 for i in range(1001): # We are using a prime number to get a non-correlatable distribution # of file sizes in the buckets size = 1048583 * i file_list.append( _FileItem("drwxrwxrwt", size, filedate, "tmp%08d" % i)) total_size += size # Test the _fill_buckets internal method with only one worker: # the result must be a bucket with the same list passed as argument rcc = RsyncCopyController(workers=1) buckets = list(rcc._fill_buckets(file_list)) assert len(buckets) == 1 assert buckets[0] == file_list # Test the _fill_buckets internal method with multiple workers # the result must be a bucket with the same list passed as argument for workers in range(2, 17): rcc = RsyncCopyController(workers=workers) buckets = list(rcc._fill_buckets(file_list)) # There is enough buckets to contains all the files assert len(buckets) >= int(total_size / BUCKET_SIZE) for i, bucket in enumerate(buckets): size = sum([f.size for f in bucket]) # The bucket is not bigger than BUCKET_SIZE assert size < BUCKET_SIZE, "Bucket %s (%s) size %s too big" % ( i, workers, size, ) # The bucket cannot be empty assert len(bucket), "Bucket %s (%s) is empty" % (i, workers)
def analyse_func(item): l = item.label item.dir_file = l + '_dir_file' item.exclude_and_protect_file = l + '_exclude_and_protect_file' item.safe_list = [_FileItem('mode', 1, 'date', 'path')] item.check_list = [_FileItem('mode', 1, 'date', 'path')]
def test_analyze_directory(self, list_files_mock, rsync_factory_mock, tmpdir): """ Unit test for RsyncCopyController._analyze_directory's code """ # Build file list for ref ref_list = [ _FileItem( 'drwxrwxrwt', 69632, datetime(year=2015, month=2, day=9, hour=15, minute=1, second=0, tzinfo=dateutil.tz.tzlocal()), '.'), _FileItem( 'drwxrwxrwt', 69612, datetime(year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal()), 'tmp'), _FileItem( '-rw-r--r--', 69632, datetime(year=2015, month=2, day=20, hour=18, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/safe'), _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/check'), _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_time'), _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_size'), ] # Build the list for source adding a new file, ... src_list = ref_list + [ _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=22, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/new'), ] # ... changing the timestamp one old file ... src_list[4] = _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=20, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_time') # ... and changing the size of another src_list[5] = _FileItem( '-rw-r--r--', 77777, datetime(year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_size') # Apply it to _list_files calls list_files_mock.side_effect = [ref_list, src_list] # Build the prerequisites server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor # Create the RsyncCopyController putting the safe_horizon between # the tmp/safe and tmp2/check timestamps rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=datetime(year=2015, month=2, day=20, hour=19, minute=0, second=0, tzinfo=dateutil.tz.tzlocal())) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Add a temp dir (usually created by copy method rcc.temp_dir = tmpdir.mkdir('tmp').strpath # Create an item to inspect item = _RsyncCopyItem(label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), is_directory=True, item_class=rcc.PGDATA_CLASS, optional=False) # Then run the _analyze_directory method rcc._analyze_directory(item) # Verify that _rsync_factory has been called correctly assert rsync_factory_mock.mock_calls == [ mock.call(item), ] # Verify that _list_files has been called correctly assert list_files_mock.mock_calls == [ mock.call(rsync_factory_mock.return_value, backup_info.get_data_directory() + '/'), mock.call(rsync_factory_mock.return_value, ':/pg/data/') ] # Check the result # 1) The list of directories should be there and should contain all # the directories assert item.dir_file assert open(item.dir_file).read() == ('.\n' 'tmp\n') # The exclude_and_protect file should be populated correctly with all # the files in the source assert item.exclude_and_protect_file assert open( item.exclude_and_protect_file).read() == ('P tmp/safe\n' '- tmp/safe\n' 'P tmp/check\n' '- tmp/check\n' 'P tmp/diff_time\n' '- tmp/diff_time\n' 'P tmp/diff_size\n' '- tmp/diff_size\n' 'P tmp/new\n' '- tmp/new\n') # The check list must contain identical files after the safe_horizon assert len(item.check_list) == 1 assert item.check_list[0].path == 'tmp/check' # The safe list must contain every file that is not in check and is # present in the source assert len(item.safe_list) == 4 assert item.safe_list[0].path == 'tmp/safe' assert item.safe_list[1].path == 'tmp/diff_time' assert item.safe_list[2].path == 'tmp/diff_size' assert item.safe_list[3].path == 'tmp/new'
def test_analyze_directory_empty_dst(self, list_files_mock, tmpdir): """ Verify that RsyncCopyController._analyze_directory produces an empty exclude_and_protect_file when the destination directory is empty. """ # Only the current directory is in file list ref_list = [ _FileItem( "drwxrwxrwt", 69632, datetime( year=2015, month=2, day=9, hour=15, minute=1, second=0, tzinfo=dateutil.tz.tzlocal(), ), ".", ), ] # Minimal src_list so that there is something to copy src_list = ref_list + [ _FileItem( "drwxrwxrwt", 69612, datetime( year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal(), ), "tmp", ), ] # Set up prerequisites and run the analyze directory function item, backup_info = self._run_analyze_directory( list_files_mock, tmpdir, ref_list, src_list) # Verify that _list_files has been called correctly assert list_files_mock.mock_calls == [ mock.call(item, backup_info.get_data_directory() + "/"), mock.call(item, ":/pg/data/"), ] # Check the result # 1) The list of directories should be there and should contain all # the directories assert item.dir_file assert open(item.dir_file).read() == (".\ntmp\n") # The exclude_and_protect file should contain only wildcards to include # all directories and exclude all files assert item.exclude_and_protect_file assert open(item.exclude_and_protect_file).read() == "+ */\n- *\n"
def test_analyze_directory(self, list_files_mock, tmpdir): """ Unit test for RsyncCopyController._analyze_directory's code """ # Build file list for ref ref_list = [ _FileItem( "drwxrwxrwt", 69632, datetime( year=2015, month=2, day=9, hour=15, minute=1, second=0, tzinfo=dateutil.tz.tzlocal(), ), ".", ), _FileItem( "drwxrwxrwt", 69612, datetime( year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal(), ), "tmp", ), _FileItem( "-rw-r--r--", 69632, datetime( year=2015, month=2, day=20, hour=18, minute=15, second=33, tzinfo=dateutil.tz.tzlocal(), ), "tmp/safe", ), _FileItem( "-rw-r--r--", 69612, datetime( year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal(), ), "tmp/check", ), _FileItem( "-rw-r--r--", 69612, datetime( year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal(), ), "tmp/diff_time", ), _FileItem( "-rw-r--r--", 69612, datetime( year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal(), ), "tmp/diff_size", ), ] # Build the list for source adding a new file, ... src_list = ref_list + [ _FileItem( "-rw-r--r--", 69612, datetime( year=2015, month=2, day=20, hour=22, minute=15, second=33, tzinfo=dateutil.tz.tzlocal(), ), "tmp/new", ), ] # ... changing the timestamp one old file ... src_list[4] = _FileItem( "-rw-r--r--", 69612, datetime( year=2015, month=2, day=20, hour=20, minute=15, second=33, tzinfo=dateutil.tz.tzlocal(), ), "tmp/diff_time", ) # ... and changing the size of another src_list[5] = _FileItem( "-rw-r--r--", 77777, datetime( year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal(), ), "tmp/diff_size", ) item, backup_info = self._run_analyze_directory( list_files_mock, tmpdir, ref_list, src_list) # Verify that _list_files has been called correctly assert list_files_mock.mock_calls == [ mock.call(item, backup_info.get_data_directory() + "/"), mock.call(item, ":/pg/data/"), ] # Check the result # 1) The list of directories should be there and should contain all # the directories assert item.dir_file assert open(item.dir_file).read() == (".\ntmp\n") # The exclude_and_protect file should be populated correctly with all # the files in the source assert item.exclude_and_protect_file assert open( item.exclude_and_protect_file).read() == ("P /tmp/safe\n" "- /tmp/safe\n" "P /tmp/check\n" "- /tmp/check\n" "P /tmp/diff_time\n" "- /tmp/diff_time\n" "P /tmp/diff_size\n" "- /tmp/diff_size\n" "P /tmp/new\n" "- /tmp/new\n") # The check list must contain identical files after the safe_horizon assert len(item.check_list) == 1 assert item.check_list[0].path == "tmp/check" # The safe list must contain every file that is not in check and is # present in the source assert len(item.safe_list) == 4 assert item.safe_list[0].path == "tmp/safe" assert item.safe_list[1].path == "tmp/diff_time" assert item.safe_list[2].path == "tmp/diff_size" assert item.safe_list[3].path == "tmp/new"
def test_list_files(self, rsync_factory_mock): """ Unit test for RsyncCopyController._list_file's code """ # Mock rsync invocation rsync_mock = mock.Mock(name="Rsync()") rsync_mock.ret = 0 rsync_mock.out = ( "drwxrwxrwt 69632 2015/02/09 15:01:00 tmp\n" "drwxrwxrwt 69612 Thu Feb 19 15:01:22 2015 tmp2") rsync_mock.err = "err" # Mock _rsync_factory() invocation rsync_factory_mock.return_value = rsync_mock # Create an item to inspect item = _RsyncCopyItem( label="pgdata", src=":/pg/data/", dst="/some/dir", is_directory=True, item_class=RsyncCopyController.PGDATA_CLASS, optional=False, ) # Test the _list_files internal method rcc = RsyncCopyController() return_values = list(rcc._list_files(item, "some/path")) # Returned list must contain two elements assert len(return_values) == 2 # Verify that _rsync_factory has been called correctly assert rsync_factory_mock.mock_calls == [ mock.call(item), ] # Check rsync.get_output has called correctly rsync_mock.get_output.assert_called_with("--no-human-readable", "--list-only", "-r", "some/path", check=True) # Check the result assert return_values[0] == _FileItem( "drwxrwxrwt", 69632, datetime( year=2015, month=2, day=9, hour=15, minute=1, second=0, tzinfo=dateutil.tz.tzlocal(), ), "tmp", ) assert return_values[1] == _FileItem( "drwxrwxrwt", 69612, datetime( year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal(), ), "tmp2", ) # Test the _list_files internal method with a wrong output (added TZ) rsync_mock.out = "drwxrwxrwt 69612 Thu Feb 19 15:01:22 CET 2015 tmp2\n" rcc = RsyncCopyController() with pytest.raises(RsyncListFilesFailure): # The list() call is needed to consume the generator list(rcc._list_files(rsync_mock, "some/path")) # Check rsync.get_output has called correctly rsync_mock.get_output.assert_called_with("--no-human-readable", "--list-only", "-r", "some/path", check=True)
def analyse_func(item): label = item.label item.dir_file = label + "_dir_file" item.exclude_and_protect_file = label + "_exclude_and_protect_file" item.safe_list = [_FileItem("mode", 1, "date", "path")] item.check_list = [_FileItem("mode", 1, "date", "path")]