def test_statistics(self, signal_mock, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir, workers): """ Unit test for RsyncCopyController.statistics's code """ # Do a fake copy run to populate the start/stop timestamps. # The steps are the same of the full run test tempdir = tmpdir.mkdir('tmp') tempfile_mock.return_value = tempdir.strpath server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None, workers=workers) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = '' rsync_mock.return_value.err = '' rsync_mock.return_value.ret = 0 # Mock analyze directory def analyse_func(item): l = item.label item.dir_file = l + '_dir_file' item.exclude_and_protect_file = l + '_exclude_and_protect_file' item.safe_list = [_FileItem('mode', 1, 'date', 'path')] item.check_list = [_FileItem('mode', 1, 'date', 'path')] analyse_mock.side_effect = analyse_func rcc.add_directory(label='tbs1', src=':/fake/location/', dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS) rcc.add_directory(label='tbs2', src=':/another/location/', dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS) rcc.add_directory( label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']) rcc.add_file(label='pg_control', src=':/pg/data/global/pg_control', dst='%s/global/pg_control' % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS) rcc.add_file(label='config_file', src=':/etc/postgresql.conf', dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False) # Do the fake run rcc.copy() # Calculate statistics result = rcc.statistics() # We cannot check the actual result because it is not predictable, # so we check that every value is present and is a number and it is # greather than 0 assert result.get('analysis_time') > 0 assert 'analysis_time_per_item' in result for tbs in ('pgdata', 'tbs1', 'tbs2'): assert result['analysis_time_per_item'][tbs] > 0 assert result.get('copy_time') > 0 assert 'copy_time_per_item' in result assert 'serialized_copy_time_per_item' in result for tbs in ('pgdata', 'tbs1', 'tbs2', 'config_file', 'pg_control'): assert result['copy_time_per_item'][tbs] > 0 assert result['serialized_copy_time_per_item'][tbs] > 0 assert result.get('number_of_workers') == rcc.workers assert result.get('total_time') > 0
def _backup_copy(self, backup_info, dest, tablespaces=None, remote_command=None, safe_horizon=None): """ Perform the actual copy of the base backup for recovery purposes First, it copies one tablespace at a time, then the PGDATA directory. Bandwidth limitation, according to configuration, is applied in the process. TODO: manage configuration files if outside PGDATA. :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. :param datetime.datetime|None safe_horizon: anything after this time has to be checked with checksum """ # Set a ':' prefix to remote destinations dest_prefix = '' if remote_command: dest_prefix = ':' # Create the copy controller object, specific for rsync, # which will drive all the copy operations. Items to be # copied are added before executing the copy() method controller = RsyncCopyController( path=self.server.path, ssh_command=remote_command, network_compression=self.config.network_compression, safe_horizon=safe_horizon, retry_times=self.config.basebackup_retry_times, retry_sleep=self.config.basebackup_retry_sleep, workers=self.config.parallel_jobs, ) # Dictionary for paths to be excluded from rsync exclude_and_protect = [] # Process every tablespace if backup_info.tablespaces: for tablespace in backup_info.tablespaces: # By default a tablespace goes in the same location where # it was on the source server when the backup was taken location = tablespace.location # If a relocation has been requested for this tablespace # use the user provided target directory if tablespaces and tablespace.name in tablespaces: location = tablespaces[tablespace.name] # If the tablespace location is inside the data directory, # exclude and protect it from being deleted during # the data directory copy if location.startswith(dest): exclude_and_protect += [location[len(dest):]] # Exclude and protect the tablespace from being deleted during # the data directory copy exclude_and_protect.append("/pg_tblspc/%s" % tablespace.oid) # Add the tablespace directory to the list of objects # to be copied by the controller controller.add_directory( label=tablespace.name, src='%s/' % backup_info.get_data_directory(tablespace.oid), dst=dest_prefix + location, bwlimit=self.config.get_bwlimit(tablespace), item_class=controller.TABLESPACE_CLASS) # Add the PGDATA directory to the list of objects to be copied # by the controller controller.add_directory(label='pgdata', src='%s/' % backup_info.get_data_directory(), dst=dest_prefix + dest, bwlimit=self.config.get_bwlimit(), exclude=[ '/pg_log/*', '/pg_xlog/*', '/pg_wal/*', '/postmaster.pid', '/recovery.conf', '/tablespace_map', ], exclude_and_protect=exclude_and_protect, item_class=controller.PGDATA_CLASS) # TODO: Manage different location for configuration files # TODO: that were not within the data directory # Execute the copy try: controller.copy() # TODO: Improve the exception output except CommandFailedException as e: msg = "data transfer failure" raise DataTransferFailure.from_command_error('rsync', e, msg)
def test_full_copy(self, signal_mock, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir): """ Test the execution of a full copy """ # Build the prerequisites tempdir = tmpdir.mkdir('tmp') tempfile_mock.return_value = tempdir.strpath server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = '' rsync_mock.return_value.err = '' rsync_mock.return_value.ret = 0 # Mock analyze directory def analyse_func(item): l = item.label item.dir_file = l + '_dir_file' item.exclude_and_protect_file = l + '_exclude_and_protect_file' item.safe_list = [_FileItem('mode', 1, 'date', 'path')] item.check_list = [_FileItem('mode', 1, 'date', 'path')] analyse_mock.side_effect = analyse_func rcc.add_directory(label='tbs1', src=':/fake/location/', dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS) rcc.add_directory(label='tbs2', src=':/another/location/', dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS) rcc.add_directory( label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']) rcc.add_file(label='pg_control', src=':/pg/data/global/pg_control', dst='%s/global/pg_control' % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS) rcc.add_file(label='config_file', src=':/etc/postgresql.conf', dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False) rcc.copy() # Check the order of calls to the Rsync mock assert rsync_mock.mock_calls == [ mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call( network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405'], include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()(':/etc/postgresql.conf', backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()(':/pg/data/global/pg_control', '%s/global/pg_control' % backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), ] # Check calls to _analyse_directory method assert analyse_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Check calls to _create_dir_and_purge method assert create_and_purge_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Utility function to build the file_list name def file_list_name(label, kind): return '%s/%s_%s_%s.list' % (tempdir.strpath, label, kind, os.getpid()) # Check the order of calls to the copy method # All the file_list arguments are None because the analyze part # has not really been executed assert copy_mock.mock_calls == [ mock.call(mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), checksum=False, file_list=file_list_name('tbs1', 'safe')), mock.call(mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), checksum=True, file_list=file_list_name('tbs1', 'check')), mock.call(mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), checksum=False, file_list=file_list_name('tbs2', 'safe')), mock.call(mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), checksum=True, file_list=file_list_name('tbs2', 'check')), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), checksum=False, file_list=file_list_name('pgdata', 'safe')), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), checksum=True, file_list=file_list_name('pgdata', 'check')), ]
def test_full_copy(self, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir): """ Test the execution of a full copy """ # Build the prerequisites tempdir = tmpdir.mkdir('tmp') tempfile_mock.return_value = tempdir.strpath server = build_real_server(global_conf={ 'barman_home': tmpdir.mkdir('home').strpath }) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = '' rsync_mock.return_value.err = '' rsync_mock.return_value.ret = 0 rcc.add_directory( label='tbs1', src=':/fake/location/', dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS), rcc.add_directory( label='tbs2', src=':/another/location/', dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS), rcc.add_directory( label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=['/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid'], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']), rcc.add_file( label='pg_control', src=':/pg/data/global/pg_control', dst='%s/global/pg_control' % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS), rcc.add_file( label='config_file', src=':/etc/postgresql.conf', dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False), rcc.copy(), # Check the order of calls to the Rsync mock assert rsync_mock.mock_calls == [ mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid'], exclude_and_protect=[ 'pg_tblspc/16387', 'pg_tblspc/16405'], retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()( ':/pg/data/global/pg_control', '%s/global/pg_control' % backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=['-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no'], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()( ':/etc/postgresql.conf', backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), ] # Check calls to _analyse_directory method assert analyse_mock.mock_calls == [ mock.call(item, tempdir.strpath) for item in rcc.item_list if item.is_directory ] # Check calls to _create_dir_and_purge method assert create_and_purge_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Check the order of calls to the copy method # All the file_list arguments are None because the analyze part # has not really been executed assert copy_mock.mock_calls == [ mock.call( mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), checksum=False, file_list=None), mock.call( mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), checksum=True, file_list=None), mock.call( mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), checksum=False, file_list=None), mock.call( mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), checksum=True, file_list=None), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), checksum=False, file_list=None), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), checksum=True, file_list=None), ]
def test_full_copy(self, smart_copy_mock, rsync_mock, tmpdir): """ Test the execution of a rsync copy :param rsync_mock: mock for the rsync command :param tmpdir: temporary dir """ # Build the prerequisites server = build_real_server( global_conf={'barman_home': tmpdir.mkdir('home').strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = '' rsync_mock.return_value.err = '' rsync_mock.return_value.ret = 0 rcc.add_directory(label='tbs1', src=':/fake/location/', dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS), rcc.add_directory(label='tbs2', src=':/another/location/', dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS), rcc.add_directory( label='pgdata', src=':/pg/data/', dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405']), rcc.add_file(label='pg_control', src=':/pg/data/global/pg_control', dst='%s/global/pg_control' % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS), rcc.add_file(label='config_file', src=':/etc/postgresql.conf', dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False), rcc.copy(), assert rsync_mock.mock_calls == [ mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call( network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid' ], exclude_and_protect=['pg_tblspc/16387', 'pg_tblspc/16405'], retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()(':/pg/data/global/pg_control', '%s/global/pg_control' % backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), mock.call(network_compression=False, args=['--itemize-changes', '--itemize-changes'], bwlimit=None, ssh='ssh', path=None, ssh_options=[ '-c', '"arcfour"', '-p', '22', '*****@*****.**', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no' ], exclude=None, exclude_and_protect=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY), mock.call()(':/etc/postgresql.conf', backup_info.get_data_directory(), allowed_retval=(0, 23, 24)), ] assert smart_copy_mock.mock_calls == [ mock.call(mock.ANY, ':/fake/location/', backup_info.get_data_directory(16387), None, None), mock.call(mock.ANY, ':/another/location/', backup_info.get_data_directory(16405), None, None), mock.call(mock.ANY, ':/pg/data/', backup_info.get_data_directory(), None, None), ]
def test_statistics( self, signal_mock, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir, workers, ): """ Unit test for RsyncCopyController.statistics's code """ # Do a fake copy run to populate the start/stop timestamps. # The steps are the same of the full run test tempdir = tmpdir.mkdir("tmp") tempfile_mock.return_value = tempdir.strpath server = build_real_server( global_conf={"barman_home": tmpdir.mkdir("home").strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None, workers=workers, ) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28, ) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = "" rsync_mock.return_value.err = "" rsync_mock.return_value.ret = 0 # Mock analyze directory def analyse_func(item): label = item.label item.dir_file = label + "_dir_file" item.exclude_and_protect_file = label + "_exclude_and_protect_file" item.safe_list = [_FileItem("mode", 1, "date", "path")] item.check_list = [_FileItem("mode", 1, "date", "path")] analyse_mock.side_effect = analyse_func rcc.add_directory( label="tbs1", src=":/fake/location/", dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS, ) rcc.add_directory( label="tbs2", src=":/another/location/", dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS, ) rcc.add_directory( label="pgdata", src=":/pg/data/", dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ "/pg_xlog/*", "/pg_log/*", "/log/*", "/recovery.conf", "/postmaster.pid", ], exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"], ) rcc.add_file( label="pg_control", src=":/pg/data/global/pg_control", dst="%s/global/pg_control" % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS, ) rcc.add_file( label="config_file", src=":/etc/postgresql.conf", dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False, ) # Do the fake run rcc.copy() # Calculate statistics result = rcc.statistics() # We cannot check the actual result because it is not predictable, # so we check that every value is present and is a number and it is # greater than 0 assert result.get("analysis_time") > 0 assert "analysis_time_per_item" in result for tbs in ("pgdata", "tbs1", "tbs2"): assert result["analysis_time_per_item"][tbs] > 0 assert result.get("copy_time") > 0 assert "copy_time_per_item" in result assert "serialized_copy_time_per_item" in result for tbs in ("pgdata", "tbs1", "tbs2", "config_file", "pg_control"): assert result["copy_time_per_item"][tbs] > 0 assert result["serialized_copy_time_per_item"][tbs] > 0 assert result.get("number_of_workers") == rcc.workers assert result.get("total_time") > 0
def test_full_copy( self, signal_mock, tempfile_mock, copy_mock, create_and_purge_mock, analyse_mock, rsync_mock, tmpdir, ): """ Test the execution of a full copy """ # Build the prerequisites tempdir = tmpdir.mkdir("tmp") tempfile_mock.return_value = tempdir.strpath server = build_real_server( global_conf={"barman_home": tmpdir.mkdir("home").strpath}) config = server.config executor = server.backup_manager.executor rcc = RsyncCopyController( path=server.path, ssh_command=executor.ssh_command, ssh_options=executor.ssh_options, network_compression=config.network_compression, reuse_backup=None, safe_horizon=None, ) backup_info = build_test_backup_info( server=server, pgdata="/pg/data", config_file="/etc/postgresql.conf", hba_file="/pg/data/pg_hba.conf", ident_file="/pg/data/pg_ident.conf", begin_xlog="0/2000028", begin_wal="000000010000000000000002", begin_offset=28, ) backup_info.save() # This is to check that all the preparation is done correctly assert os.path.exists(backup_info.filename) # Silence the access to result properties rsync_mock.return_value.out = "" rsync_mock.return_value.err = "" rsync_mock.return_value.ret = 0 # Mock analyze directory def analyse_func(item): label = item.label item.dir_file = label + "_dir_file" item.exclude_and_protect_file = label + "_exclude_and_protect_file" item.safe_list = [_FileItem("mode", 1, "date", "path")] item.check_list = [_FileItem("mode", 1, "date", "path")] analyse_mock.side_effect = analyse_func rcc.add_directory( label="tbs1", src=":/fake/location/", dst=backup_info.get_data_directory(16387), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS, ) rcc.add_directory( label="tbs2", src=":/another/location/", dst=backup_info.get_data_directory(16405), reuse=None, bwlimit=None, item_class=rcc.TABLESPACE_CLASS, ) rcc.add_directory( label="pgdata", src=":/pg/data/", dst=backup_info.get_data_directory(), reuse=None, bwlimit=None, item_class=rcc.PGDATA_CLASS, exclude=[ "/pg_xlog/*", "/pg_log/*", "/log/*", "/recovery.conf", "/postmaster.pid", ], exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"], ) rcc.add_file( label="pg_control", src=":/pg/data/global/pg_control", dst="%s/global/pg_control" % backup_info.get_data_directory(), item_class=rcc.PGCONTROL_CLASS, ) rcc.add_file( label="config_file", src=":/etc/postgresql.conf", dst=backup_info.get_data_directory(), item_class=rcc.CONFIG_CLASS, optional=False, ) rcc.copy() # Check the order of calls to the Rsync mock assert rsync_mock.mock_calls == [ mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=[ "/pg_xlog/*", "/pg_log/*", "/log/*", "/recovery.conf", "/postmaster.pid", ], exclude_and_protect=["pg_tblspc/16387", "pg_tblspc/16405"], include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call()( ":/etc/postgresql.conf", backup_info.get_data_directory(), allowed_retval=(0, 23, 24), ), mock.call( network_compression=False, args=[ "--ignore-missing-args", "--itemize-changes", "--itemize-changes", ], bwlimit=None, ssh="ssh", path=None, ssh_options=[ "-c", '"arcfour"', "-p", "22", "*****@*****.**", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", ], exclude=None, exclude_and_protect=None, include=None, retry_sleep=0, retry_times=0, retry_handler=mock.ANY, ), mock.call()( ":/pg/data/global/pg_control", "%s/global/pg_control" % backup_info.get_data_directory(), allowed_retval=(0, 23, 24), ), ] # Check calls to _analyse_directory method assert analyse_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Check calls to _create_dir_and_purge method assert create_and_purge_mock.mock_calls == [ mock.call(item) for item in rcc.item_list if item.is_directory ] # Utility function to build the file_list name def file_list_name(label, kind): return "%s/%s_%s_%s.list" % (tempdir.strpath, label, kind, os.getpid()) # Check the order of calls to the copy method # All the file_list arguments are None because the analyze part # has not really been executed assert copy_mock.mock_calls == [ mock.call( mock.ANY, ":/fake/location/", backup_info.get_data_directory(16387), checksum=False, file_list=file_list_name("tbs1", "safe"), ), mock.call( mock.ANY, ":/fake/location/", backup_info.get_data_directory(16387), checksum=True, file_list=file_list_name("tbs1", "check"), ), mock.call( mock.ANY, ":/another/location/", backup_info.get_data_directory(16405), checksum=False, file_list=file_list_name("tbs2", "safe"), ), mock.call( mock.ANY, ":/another/location/", backup_info.get_data_directory(16405), checksum=True, file_list=file_list_name("tbs2", "check"), ), mock.call( mock.ANY, ":/pg/data/", backup_info.get_data_directory(), checksum=False, file_list=file_list_name("pgdata", "safe"), ), mock.call( mock.ANY, ":/pg/data/", backup_info.get_data_directory(), checksum=True, file_list=file_list_name("pgdata", "check"), ), ]
def _backup_copy(self, backup_info, dest, tablespaces=None, remote_command=None, safe_horizon=None): """ Perform the actual copy of the base backup for recovery purposes First, it copies one tablespace at a time, then the PGDATA directory. Bandwidth limitation, according to configuration, is applied in the process. TODO: manage configuration files if outside PGDATA. :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. :param datetime.datetime|None safe_horizon: anything after this time has to be checked with checksum """ # Set a ':' prefix to remote destinations dest_prefix = '' if remote_command: dest_prefix = ':' # Create the copy controller object, specific for rsync, # which will drive all the copy operations. Items to be # copied are added before executing the copy() method controller = RsyncCopyController( path=self.server.path, ssh_command=remote_command, network_compression=self.config.network_compression, safe_horizon=safe_horizon, retry_times=self.config.basebackup_retry_times, retry_sleep=self.config.basebackup_retry_sleep, ) # Dictionary for paths to be excluded from rsync exclude_and_protect = [] # Process every tablespace if backup_info.tablespaces: for tablespace in backup_info.tablespaces: # By default a tablespace goes in the same location where # it was on the source server when the backup was taken location = tablespace.location # If a relocation has been requested for this tablespace # use the user provided target directory if tablespaces and tablespace.name in tablespaces: location = tablespaces[tablespace.name] # If the tablespace location is inside the data directory, # exclude and protect it from being deleted during # the data directory copy if location.startswith(dest): exclude_and_protect += [location[len(dest):]] # Exclude and protect the tablespace from being deleted during # the data directory copy exclude_and_protect.append("/pg_tblspc/%s" % tablespace.oid) # Add the tablespace directory to the list of objects # to be copied by the controller controller.add_directory( label=tablespace.name, src='%s/' % backup_info.get_data_directory(tablespace.oid), dst=dest_prefix + location, bwlimit=self.config.get_bwlimit(tablespace), item_class=controller.TABLESPACE_CLASS ) # Add the PGDATA directory to the list of objects to be copied # by the controller controller.add_directory( label='pgdata', src='%s/' % backup_info.get_data_directory(), dst=dest_prefix + dest, bwlimit=self.config.get_bwlimit(), exclude=[ '/pg_xlog/*', '/pg_log/*', '/recovery.conf', '/postmaster.pid'], exclude_and_protect=exclude_and_protect, item_class=controller.PGDATA_CLASS ) # TODO: Manage different location for configuration files # TODO: that were not within the data directory # Execute the copy try: controller.copy() # TODO: Improve the exception output except CommandFailedException as e: msg = "data transfer failure" raise DataTransferFailure.from_command_error( 'rsync', e, msg)