def test_list_files(self): """ Unit test for RsyncCopyController_list_file's code """ # Mock rsync invocation rsync_mock = mock.Mock(name='Rsync()') rsync_mock.ret = 0 rsync_mock.out = 'drwxrwxrwt 69632 2015/02/09 15:01:00 tmp\n' \ 'drwxrwxrwt 69612 2015/02/19 15:01:22 tmp2' rsync_mock.err = 'err' # Test the _list_files internal method rcc = RsyncCopyController() return_values = list(rcc._list_files(rsync_mock, 'some/path')) # Returned list must contain two elements assert len(return_values) == 2 # Check rsync.get_output has called correctly rsync_mock.get_output.assert_called_with( '--no-human-readable', '--list-only', '-r', 'some/path', check=True) # Check the result assert return_values[0] == rcc._FileItem( 'drwxrwxrwt', 69632, datetime(year=2015, month=2, day=9, hour=15, minute=1, second=0, tzinfo=dateutil.tz.tzlocal()), 'tmp') assert return_values[1] == rcc._FileItem( 'drwxrwxrwt', 69612, datetime(year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal()), 'tmp2')
def test_reuse_args(self): """ Simple test for the _reuse_args method The method is necessary for the execution of incremental backups, we need to test that the method build correctly the rsync option that enables the incremental backup """ # Build the prerequisites backup_manager = build_backup_manager() server = backup_manager.server config = server.config executor = server.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None) reuse_dir = "some/dir" # Test for disabled incremental assert rcc._reuse_args(reuse_dir) == [] # Test for link incremental rcc.reuse_backup = 'link' assert rcc._reuse_args(reuse_dir) == \ ['--link-dest=some/dir'] # Test for copy incremental rcc.reuse_backup = 'copy' assert rcc._reuse_args(reuse_dir) == \ ['--copy-dest=some/dir']
def test_full_copy(self, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir): """ Test the execution of a full copy """ # Build the prerequisites tempdir = tmpdir.mkdir('tmp') tempfile_mock.return_value = tempdir.strpath server = build_real_server(global_conf={ 'barman_home': tmpdir.mkdir('home').strpath }) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = '' rsync_mock.return_value.err = '' rsync_mock.return_value.ret = 0 rcc.add_directory( label='tbs1', src=':/fake/location/', dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS), rcc.add_directory( label='tbs2', src=':/another/location/', dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS), rcc.add_directory( label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=['/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid'], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']), rcc.add_file( label='pg_control', src=':/pg/data/global/pg_control', dst='%s/global/pg_control' % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS), rcc.add_file( label='config_file', src=':/etc/postgresql.conf', dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False), rcc.copy(), # Check the order of calls to the Rsync mock assert rsync_mock.mock_calls == [ mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid'], exclude_and_protect=[ 'pg_tblspc/16387', 'pg_tblspc/16405'], retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()( ':/pg/data/global/pg_control', '%s/global/pg_control' % backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()( ':/etc/postgresql.conf', backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), ] # Check calls to _analyse_directory method assert analyse_mock.mock_calls == [ mock.call(item, tempdir.strpath) for item in rcc.item_list if item.is_directory ] # Check calls to _create_dir_and_purge method assert create_and_purge_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Check the order of calls to the copy method # All the file_list arguments are None because the analyze part # has not really been executed assert copy_mock.mock_calls == [ mock.call( mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), checksum=False, file_list=None), mock.call( mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), checksum=True, file_list=None), mock.call( mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), checksum=False, file_list=None), mock.call( mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), checksum=True, file_list=None), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), checksum=False, file_list=None), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), checksum=True, file_list=None), ]
def test_statistics(self, signal_mock, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir, workers): """ Unit test for RsyncCopyController.statistics's code """ # Do a fake copy run to populate the start/stop timestamps. # The steps are the same of the full run test tempdir = tmpdir.mkdir('tmp') tempfile_mock.return_value = tempdir.strpath server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None, workers=workers) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = '' rsync_mock.return_value.err = '' rsync_mock.return_value.ret = 0 # Mock analyze directory def analyse_func(item): l = item.label item.dir_file = l + '_dir_file' item.exclude_and_protect_file = l + '_exclude_and_protect_file' item.safe_list = [_FileItem('mode', 1, 'date', 'path')] item.check_list = [_FileItem('mode', 1, 'date', 'path')] analyse_mock.side_effect = analyse_func rcc.add_directory(label='tbs1', src=':/fake/location/', dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS) rcc.add_directory(label='tbs2', src=':/another/location/', dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS) rcc.add_directory( label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']) rcc.add_file(label='pg_control', src=':/pg/data/global/pg_control', dst='%s/global/pg_control' % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS) rcc.add_file(label='config_file', src=':/etc/postgresql.conf', dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False) # Do the fake run rcc.copy() # Calculate statistics result = rcc.statistics() # We cannot check the actual result because it is not predictable, # so we check that every value is present and is a number and it is # greather than 0 assert result.get('analysis_time') > 0 assert 'analysis_time_per_item' in result for tbs in ('pgdata', 'tbs1', 'tbs2'): assert result['analysis_time_per_item'][tbs] > 0 assert result.get('copy_time') > 0 assert 'copy_time_per_item' in result assert 'serialized_copy_time_per_item' in result for tbs in ('pgdata', 'tbs1', 'tbs2', 'config_file', 'pg_control'): assert result['copy_time_per_item'][tbs] > 0 assert result['serialized_copy_time_per_item'][tbs] > 0 assert result.get('number_of_workers') == rcc.workers assert result.get('total_time') > 0
def _backup_copy(self, backup_info, dest, tablespaces=None, remote_command=None, safe_horizon=None): """ Perform the actual copy of the base backup for recovery purposes First, it copies one tablespace at a time, then the PGDATA directory. Bandwidth limitation, according to configuration, is applied in the process. TODO: manage configuration files if outside PGDATA. :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. :param datetime.datetime|None safe_horizon: anything after this time has to be checked with checksum """ # Set a ':' prefix to remote destinations dest_prefix = '' if remote_command: dest_prefix = ':' # Create the copy controller object, specific for rsync, # which will drive all the copy operations. Items to be # copied are added before executing the copy() method controller = RsyncCopyController( path=self.server.path, ssh_command=remote_command, network_compression=self.config.network_compression, safe_horizon=safe_horizon, retry_times=self.config.basebackup_retry_times, retry_sleep=self.config.basebackup_retry_sleep, workers=self.config.parallel_jobs, ) # Dictionary for paths to be excluded from rsync exclude_and_protect = [] # Process every tablespace if backup_info.tablespaces: for tablespace in backup_info.tablespaces: # By default a tablespace goes in the same location where # it was on the source server when the backup was taken location = tablespace.location # If a relocation has been requested for this tablespace # use the user provided target directory if tablespaces and tablespace.name in tablespaces: location = tablespaces[tablespace.name] # If the tablespace location is inside the data directory, # exclude and protect it from being deleted during # the data directory copy if location.startswith(dest): exclude_and_protect += [location[len(dest):]] # Exclude and protect the tablespace from being deleted during # the data directory copy exclude_and_protect.append("/pg_tblspc/%s" % tablespace.oid) # Add the tablespace directory to the list of objects # to be copied by the controller controller.add_directory( label=tablespace.name, src='%s/' % backup_info.get_data_directory(tablespace.oid), dst=dest_prefix + location, bwlimit=self.config.get_bwlimit(tablespace), item_class=controller.TABLESPACE_CLASS) # Add the PGDATA directory to the list of objects to be copied # by the controller controller.add_directory(label='pgdata', src='%s/' % backup_info.get_data_directory(), dst=dest_prefix + dest, bwlimit=self.config.get_bwlimit(), exclude=[ '/pg_log/*', '/pg_xlog/*', '/postmaster.pid', '/recovery.conf', '/tablespace_map', ], exclude_and_protect=exclude_and_protect, item_class=controller.PGDATA_CLASS) # TODO: Manage different location for configuration files # TODO: that were not within the data directory # Execute the copy try: controller.copy() # TODO: Improve the exception output except CommandFailedException as e: msg = "data transfer failure" raise DataTransferFailure.from_command_error('rsync', e, msg)
def test_create_dir_and_purge(self, rsync_ignore_mock, rsync_factory_mock, tmpdir): """ Unit test for RsyncCopyController._create_dir_and_purge's code """ # Build the prerequisites server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor # Create the RsyncCopyController putting the safe_horizon between # the tmp/safe and tmp2/check timestamps rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=datetime(year=2015, month=2, day=20, hour=19, minute=0, second=0, tzinfo=dateutil.tz.tzlocal())) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Create an item to inspect item = _RsyncCopyItem(label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), is_directory=True, item_class=rcc.PGDATA_CLASS, optional=False) # Then run the _create_dir_and_purge method rcc._create_dir_and_purge(item) # Verify that _rsync_factory has been called correctly assert rsync_factory_mock.mock_calls == [ mock.call(item), ] # Verify that _rsync_ignore_vanished_files has been called correctly assert rsync_ignore_mock.mock_calls == [ mock.call(rsync_factory_mock.return_value, '--recursive', '--delete', '--files-from=None', '--filter', 'merge None', ':/pg/data/', backup_info.get_data_directory(), check=True), ]
def test_copy(self, rsync_ignore_mock, tmpdir): """ Unit test for RsyncCopyController._copy's code """ # Build the prerequisites server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor # Create the RsyncCopyController putting the safe_horizon between # the tmp/safe and tmp2/check timestamps rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=datetime(year=2015, month=2, day=20, hour=19, minute=0, second=0, tzinfo=dateutil.tz.tzlocal())) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Create an rsync mock rsync_mock = mock.Mock(name='Rsync()') # Then run the _copy method rcc._copy(rsync_mock, ':/pg/data/', backup_info.get_data_directory(), '/path/to/file.list', checksum=True) # Verify that _rsync_ignore_vanished_files has been called correctly assert rsync_ignore_mock.mock_calls == [ mock.call(rsync_mock, ':/pg/data/', backup_info.get_data_directory(), '--files-from=/path/to/file.list', '--checksum', check=True), ] # Try again without checksum rsync_ignore_mock.reset_mock() rcc._copy(rsync_mock, ':/pg/data/', backup_info.get_data_directory(), '/path/to/file.list', checksum=False) # Verify that _rsync_ignore_vanished_files has been called correctly assert rsync_ignore_mock.mock_calls == [ mock.call(rsync_mock, ':/pg/data/', backup_info.get_data_directory(), '--files-from=/path/to/file.list', check=True), ]
def test_full_copy(self, signal_mock, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir): """ Test the execution of a full copy """ # Build the prerequisites tempdir = tmpdir.mkdir('tmp') tempfile_mock.return_value = tempdir.strpath server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = '' rsync_mock.return_value.err = '' rsync_mock.return_value.ret = 0 # Mock analyze directory def analyse_func(item): l = item.label item.dir_file = l + '_dir_file' item.exclude_and_protect_file = l + '_exclude_and_protect_file' item.safe_list = [_FileItem('mode', 1, 'date', 'path')] item.check_list = [_FileItem('mode', 1, 'date', 'path')] analyse_mock.side_effect = analyse_func rcc.add_directory(label='tbs1', src=':/fake/location/', dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS) rcc.add_directory(label='tbs2', src=':/another/location/', dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS) rcc.add_directory( label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']) rcc.add_file(label='pg_control', src=':/pg/data/global/pg_control', dst='%s/global/pg_control' % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS) rcc.add_file(label='config_file', src=':/etc/postgresql.conf', dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False) rcc.copy() # Check the order of calls to the Rsync mock assert rsync_mock.mock_calls == [ mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call( network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405'], include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()(':/etc/postgresql.conf', backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()(':/pg/data/global/pg_control', '%s/global/pg_control' % backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), ] # Check calls to _analyse_directory method assert analyse_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Check calls to _create_dir_and_purge method assert create_and_purge_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Utility function to build the file_list name def file_list_name(label, kind): return '%s/%s_%s_%s.list' % (tempdir.strpath, label, kind, os.getpid()) # Check the order of calls to the copy method # All the file_list arguments are None because the analyze part # has not really been executed assert copy_mock.mock_calls == [ mock.call(mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), checksum=False, file_list=file_list_name('tbs1', 'safe')), mock.call(mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), checksum=True, file_list=file_list_name('tbs1', 'check')), mock.call(mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), checksum=False, file_list=file_list_name('tbs2', 'safe')), mock.call(mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), checksum=True, file_list=file_list_name('tbs2', 'check')), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), checksum=False, file_list=file_list_name('pgdata', 'safe')), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), checksum=True, file_list=file_list_name('pgdata', 'check')), ]
def test_analyze_directory(self, list_files_mock, rsync_factory_mock, tmpdir): """ Unit test for RsyncCopyController._analyze_directory's code """ # Build file list for ref ref_list = [ _FileItem( 'drwxrwxrwt', 69632, datetime(year=2015, month=2, day=9, hour=15, minute=1, second=0, tzinfo=dateutil.tz.tzlocal()), '.'), _FileItem( 'drwxrwxrwt', 69612, datetime(year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal()), 'tmp'), _FileItem( '-rw-r--r--', 69632, datetime(year=2015, month=2, day=20, hour=18, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/safe'), _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/check'), _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_time'), _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_size'), ] # Build the list for source adding a new file, ... src_list = ref_list + [ _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=22, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/new'), ] # ... changing the timestamp one old file ... src_list[4] = _FileItem( '-rw-r--r--', 69612, datetime(year=2015, month=2, day=20, hour=20, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_time') # ... and changing the size of another src_list[5] = _FileItem( '-rw-r--r--', 77777, datetime(year=2015, month=2, day=20, hour=19, minute=15, second=33, tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_size') # Apply it to _list_files calls list_files_mock.side_effect = [ref_list, src_list] # Build the prerequisites server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor # Create the RsyncCopyController putting the safe_horizon between # the tmp/safe and tmp2/check timestamps rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=datetime(year=2015, month=2, day=20, hour=19, minute=0, second=0, tzinfo=dateutil.tz.tzlocal())) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Add a temp dir (usually created by copy method rcc.temp_dir = tmpdir.mkdir('tmp').strpath # Create an item to inspect item = _RsyncCopyItem(label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), is_directory=True, item_class=rcc.PGDATA_CLASS, optional=False) # Then run the _analyze_directory method rcc._analyze_directory(item) # Verify that _rsync_factory has been called correctly assert rsync_factory_mock.mock_calls == [ mock.call(item), ] # Verify that _list_files has been called correctly assert list_files_mock.mock_calls == [ mock.call(rsync_factory_mock.return_value, backup_info.get_data_directory() + '/'), mock.call(rsync_factory_mock.return_value, ':/pg/data/') ] # Check the result # 1) The list of directories should be there and should contain all # the directories assert item.dir_file assert open(item.dir_file).read() == ('.\n' 'tmp\n') # The exclude_and_protect file should be populated correctly with all # the files in the source assert item.exclude_and_protect_file assert open( item.exclude_and_protect_file).read() == ('P tmp/safe\n' '- tmp/safe\n' 'P tmp/check\n' '- tmp/check\n' 'P tmp/diff_time\n' '- tmp/diff_time\n' 'P tmp/diff_size\n' '- tmp/diff_size\n' 'P tmp/new\n' '- tmp/new\n') # The check list must contain identical files after the safe_horizon assert len(item.check_list) == 1 assert item.check_list[0].path == 'tmp/check' # The safe list must contain every file that is not in check and is # present in the source assert len(item.safe_list) == 4 assert item.safe_list[0].path == 'tmp/safe' assert item.safe_list[1].path == 'tmp/diff_time' assert item.safe_list[2].path == 'tmp/diff_size' assert item.safe_list[3].path == 'tmp/new'
def test_rsync_ignore_vanished_files(self): """ Unit test for RsyncCopyController._rsync_ignore_vanished_files's code """ # Create the RsyncCopyController rcc = RsyncCopyController() # Create an rsync mock rsync_mock = mock.Mock(name='Rsync()') rsync_mock.out = 'out' rsync_mock.err = 'err' rsync_mock.ret = 0 # Then run the _copy method out, err = rcc._rsync_ignore_vanished_files(rsync_mock, 1, 2, a=3, b=4) # Verify that rsync has been called correctly assert rsync_mock.mock_calls == [ mock.call.get_output(1, 2, a=3, b=4, allowed_retval=(0, 23, 24)) ] # Verify the result assert out == rsync_mock.out assert err == rsync_mock.err # Check with return code != 0 # 24 - Partial transfer due to vanished source files rsync_mock.reset_mock() rsync_mock.ret = 24 rcc._rsync_ignore_vanished_files(rsync_mock, 1, 2, a=3, b=4) # Check with return code != 0 # 23 - Partial transfer due to error # This should raise because the error contains an invalid response rsync_mock.reset_mock() rsync_mock.ret = 23 with pytest.raises(CommandFailedException): rcc._rsync_ignore_vanished_files(rsync_mock, 1, 2, a=3, b=4) # Check with return code != 0 # 23 - Partial transfer due to error # This should not raise rsync_mock.reset_mock() rsync_mock.ret = 23 rsync_mock.err = ( # a file has vanished before rsync start 'rsync: link_stat "a/file" failed: No such file or directory (2)\n' # files which vanished after rsync start 'file has vanished: "some/other/file"\n' # files which have been truncated during transfer 'rsync: read errors mapping "/truncated": No data available (61)\n' # final summary 'rsync error: some files/attrs were not transferred ' '(see previous errors) (code 23) at main.c(1249) ' '[generator=3.0.6]\n') rcc._rsync_ignore_vanished_files(rsync_mock, 1, 2, a=3, b=4) # Check with return code != 0 # 23 - Partial transfer due to error # Version with 'receiver' as error source # This should not raise rsync_mock.reset_mock() rsync_mock.ret = 23 rsync_mock.err = ( # a file has vanished before rsync start 'rsync: link_stat "a/file" failed: No such file or directory (2)\n' # files which vanished after rsync start 'file has vanished: "some/other/file"\n' # files which have been truncated during transfer 'rsync: read errors mapping "/truncated": No data available (61)\n' # final summary 'rsync error: some files/attrs were not transferred ' '(see previous errors) (code 23) at main.c(1249) ' '[Receiver=3.1.2]\n') rcc._rsync_ignore_vanished_files(rsync_mock, 1, 2, a=3, b=4)
def test_full_copy(self, smart_copy_mock, rsync_mock, tmpdir): """ Test the execution of a rsync copy :param rsync_mock: mock for the rsync command :param tmpdir: temporary dir """ # Build the prerequisites server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = '' rsync_mock.return_value.err = '' rsync_mock.return_value.ret = 0 rcc.add_directory(label='tbs1', src=':/fake/location/', dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS), rcc.add_directory(label='tbs2', src=':/another/location/', dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS), rcc.add_directory( label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']), rcc.add_file(label='pg_control', src=':/pg/data/global/pg_control', dst='%s/global/pg_control' % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS), rcc.add_file(label='config_file', src=':/etc/postgresql.conf', dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False), rcc.copy(), assert rsync_mock.mock_calls == [ mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call( network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405'], retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()(':/pg/data/global/pg_control', '%s/global/pg_control' % backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()(':/etc/postgresql.conf', backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), ] assert smart_copy_mock.mock_calls == [ mock.call(mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), None, None), mock.call(mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), None, None), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), None, None), ]
def _run_analyze_directory(self, list_files_mock, tmpdir, ref_list, src_list): # Apply it to _list_files calls list_files_mock.side_effect = [ref_list, src_list] # Build the prerequisites server = build_real_server( global_conf={"barman_home": tmpdir.mkdir("home").strpath}) config = server.config executor = server.backup_manager.executor # Create the RsyncCopyController putting the safe_horizon between # the tmp/safe and tmp2/check timestamps rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=datetime( year=2015, month=2, day=20, hour=19, minute=0, second=0, tzinfo=dateutil.tz.tzlocal(), ), ) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28, ) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Add a temp dir (usually created by copy method rcc.temp_dir = tmpdir.mkdir("tmp").strpath # Create an item to inspect item = _RsyncCopyItem( label="pgdata", src=":/pg/data/", dst=backup_info.get_data_directory(), is_directory=True, item_class=rcc.PGDATA_CLASS, optional=False, ) # Then run the _analyze_directory method rcc._analyze_directory(item) return item, backup_info
def test_list_files(self, rsync_factory_mock): """ Unit test for RsyncCopyController._list_file's code """ # Mock rsync invocation rsync_mock = mock.Mock(name="Rsync()") rsync_mock.ret = 0 rsync_mock.out = ( "drwxrwxrwt 69632 2015/02/09 15:01:00 tmp\n" "drwxrwxrwt 69612 Thu Feb 19 15:01:22 2015 tmp2") rsync_mock.err = "err" # Mock _rsync_factory() invocation rsync_factory_mock.return_value = rsync_mock # Create an item to inspect item = _RsyncCopyItem( label="pgdata", src=":/pg/data/", dst="/some/dir", is_directory=True, item_class=RsyncCopyController.PGDATA_CLASS, optional=False, ) # Test the _list_files internal method rcc = RsyncCopyController() return_values = list(rcc._list_files(item, "some/path")) # Returned list must contain two elements assert len(return_values) == 2 # Verify that _rsync_factory has been called correctly assert rsync_factory_mock.mock_calls == [ mock.call(item), ] # Check rsync.get_output has called correctly rsync_mock.get_output.assert_called_with("--no-human-readable", "--list-only", "-r", "some/path", check=True) # Check the result assert return_values[0] == _FileItem( "drwxrwxrwt", 69632, datetime( year=2015, month=2, day=9, hour=15, minute=1, second=0, tzinfo=dateutil.tz.tzlocal(), ), "tmp", ) assert return_values[1] == _FileItem( "drwxrwxrwt", 69612, datetime( year=2015, month=2, day=19, hour=15, minute=1, second=22, tzinfo=dateutil.tz.tzlocal(), ), "tmp2", ) # Test the _list_files internal method with a wrong output (added TZ) rsync_mock.out = "drwxrwxrwt 69612 Thu Feb 19 15:01:22 CET 2015 tmp2\n" rcc = RsyncCopyController() with pytest.raises(RsyncListFilesFailure): # The list() call is needed to consume the generator list(rcc._list_files(rsync_mock, "some/path")) # Check rsync.get_output has called correctly rsync_mock.get_output.assert_called_with("--no-human-readable", "--list-only", "-r", "some/path", check=True)
def test_statistics( self, signal_mock, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir, workers, ): """ Unit test for RsyncCopyController.statistics's code """ # Do a fake copy run to populate the start/stop timestamps. # The steps are the same of the full run test tempdir = tmpdir.mkdir("tmp") tempfile_mock.return_value = tempdir.strpath server = build_real_server( global_conf={"barman_home": tmpdir.mkdir("home").strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None, workers=workers, ) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28, ) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = "" rsync_mock.return_value.err = "" rsync_mock.return_value.ret = 0 # Mock analyze directory def analyse_func(item): label = item.label item.dir_file = label + "_dir_file" item.exclude_and_protect_file = label + "_exclude_and_protect_file" item.safe_list = [_FileItem("mode", 1, "date", "path")] item.check_list = [_FileItem("mode", 1, "date", "path")] analyse_mock.side_effect = analyse_func rcc.add_directory( label="tbs1", src=":/fake/location/", dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS, ) rcc.add_directory( label="tbs2", src=":/another/location/", dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS, ) rcc.add_directory( label="pgdata", src=":/pg/data/", dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ "/pg_xlog/*", "/pg_log/*", "/log/*", "/recovery.conf", "/postmaster.pid", ], exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"], ) rcc.add_file( label="pg_control", src=":/pg/data/global/pg_control", dst="%s/global/pg_control" % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS, ) rcc.add_file( label="config_file", src=":/etc/postgresql.conf", dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False, ) # Do the fake run rcc.copy() # Calculate statistics result = rcc.statistics() # We cannot check the actual result because it is not predictable, # so we check that every value is present and is a number and it is # greater than 0 assert result.get("analysis_time") > 0 assert "analysis_time_per_item" in result for tbs in ("pgdata", "tbs1", "tbs2"): assert result["analysis_time_per_item"][tbs] > 0 assert result.get("copy_time") > 0 assert "copy_time_per_item" in result assert "serialized_copy_time_per_item" in result for tbs in ("pgdata", "tbs1", "tbs2", "config_file", "pg_control"): assert result["copy_time_per_item"][tbs] > 0 assert result["serialized_copy_time_per_item"][tbs] > 0 assert result.get("number_of_workers") == rcc.workers assert result.get("total_time") > 0
def test_full_copy( self, signal_mock, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir, ): """ Test the execution of a full copy """ # Build the prerequisites tempdir = tmpdir.mkdir("tmp") tempfile_mock.return_value = tempdir.strpath server = build_real_server( global_conf={"barman_home": tmpdir.mkdir("home").strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None, ) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28, ) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = "" rsync_mock.return_value.err = "" rsync_mock.return_value.ret = 0 # Mock analyze directory def analyse_func(item): label = item.label item.dir_file = label + "_dir_file" item.exclude_and_protect_file = label + "_exclude_and_protect_file" item.safe_list = [_FileItem("mode", 1, "date", "path")] item.check_list = [_FileItem("mode", 1, "date", "path")] analyse_mock.side_effect = analyse_func rcc.add_directory( label="tbs1", src=":/fake/location/", dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS, ) rcc.add_directory( label="tbs2", src=":/another/location/", dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS, ) rcc.add_directory( label="pgdata", src=":/pg/data/", dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ "/pg_xlog/*", "/pg_log/*", "/log/*", "/recovery.conf", "/postmaster.pid", ], exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"], ) rcc.add_file( label="pg_control", src=":/pg/data/global/pg_control", dst="%s/global/pg_control" % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS, ) rcc.add_file( label="config_file", src=":/etc/postgresql.conf", dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False, ) rcc.copy() # Check the order of calls to the Rsync mock assert rsync_mock.mock_calls == [ mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=[ "/pg_xlog/*", "/pg_log/*", "/log/*", "/recovery.conf", "/postmaster.pid", ], exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"], include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call()( ":/etc/postgresql.conf", backup_info.get_data_directory(), allowed_retval=(0, 23, 24), ), mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call()( ":/pg/data/global/pg_control", "%s/global/pg_control" % backup_info.get_data_directory(), allowed_retval=(0, 23, 24), ), ] # Check calls to _analyse_directory method assert analyse_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Check calls to _create_dir_and_purge method assert create_and_purge_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Utility function to build the file_list name def file_list_name(label, kind): return "%s/%s_%s_%s.list" % (tempdir.strpath, label, kind, os.getpid()) # Check the order of calls to the copy method # All the file_list arguments are None because the analyze part # has not really been executed assert copy_mock.mock_calls == [ mock.call( mock.ANY, ":/fake/location/", backup_info.get_data_directory(16387), checksum=False, file_list=file_list_name("tbs1", "safe"), ), mock.call( mock.ANY, ":/fake/location/", backup_info.get_data_directory(16387), checksum=True, file_list=file_list_name("tbs1", "check"), ), mock.call( mock.ANY, ":/another/location/", backup_info.get_data_directory(16405), checksum=False, file_list=file_list_name("tbs2", "safe"), ), mock.call( mock.ANY, ":/another/location/", backup_info.get_data_directory(16405), checksum=True, file_list=file_list_name("tbs2", "check"), ), mock.call( mock.ANY, ":/pg/data/", backup_info.get_data_directory(), checksum=False, file_list=file_list_name("pgdata", "safe"), ), mock.call( mock.ANY, ":/pg/data/", backup_info.get_data_directory(), checksum=True, file_list=file_list_name("pgdata", "check"), ), ]
def _backup_copy(self, backup_info, dest, tablespaces=None, remote_command=None, safe_horizon=None): """ Perform the actual copy of the base backup for recovery purposes First, it copies one tablespace at a time, then the PGDATA directory. Bandwidth limitation, according to configuration, is applied in the process. TODO: manage configuration files if outside PGDATA. :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. :param datetime.datetime|None safe_horizon: anything after this time has to be checked with checksum """ # Set a ':' prefix to remote destinations dest_prefix = '' if remote_command: dest_prefix = ':' # Create the copy controller object, specific for rsync, # which will drive all the copy operations. Items to be # copied are added before executing the copy() method controller = RsyncCopyController( path=self.server.path, ssh_command=remote_command, network_compression=self.config.network_compression, safe_horizon=safe_horizon, retry_times=self.config.basebackup_retry_times, retry_sleep=self.config.basebackup_retry_sleep, ) # Dictionary for paths to be excluded from rsync exclude_and_protect = [] # Process every tablespace if backup_info.tablespaces: for tablespace in backup_info.tablespaces: # By default a tablespace goes in the same location where # it was on the source server when the backup was taken location = tablespace.location # If a relocation has been requested for this tablespace # use the user provided target directory if tablespaces and tablespace.name in tablespaces: location = tablespaces[tablespace.name] # If the tablespace location is inside the data directory, # exclude and protect it from being deleted during # the data directory copy if location.startswith(dest): exclude_and_protect += [location[len(dest):]] # Exclude and protect the tablespace from being deleted during # the data directory copy exclude_and_protect.append("/pg_tblspc/%s" % tablespace.oid) # Add the tablespace directory to the list of objects # to be copied by the controller controller.add_directory( label=tablespace.name, src='%s/' % backup_info.get_data_directory(tablespace.oid), dst=dest_prefix + location, bwlimit=self.config.get_bwlimit(tablespace), item_class=controller.TABLESPACE_CLASS ) # Add the PGDATA directory to the list of objects to be copied # by the controller controller.add_directory( label='pgdata', src='%s/' % backup_info.get_data_directory(), dst=dest_prefix + dest, bwlimit=self.config.get_bwlimit(), exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid'], exclude_and_protect=exclude_and_protect, item_class=controller.PGDATA_CLASS ) # TODO: Manage different location for configuration files # TODO: that were not within the data directory # Execute the copy try: controller.copy() # TODO: Improve the exception output except CommandFailedException as e: msg = "data transfer failure" raise DataTransferFailure.from_command_error( 'rsync', e, msg)