def database_verify(self, data_directory):
"""
User command which finds the most recent database restore wale
info dir and invokes verification based on that. This only
works immediately after restore before any database recovery
of course.
"""
manifests_dir = manifest.directory(data_directory)
if not os.path.isdir(manifests_dir):
raise UserException(
msg='Did not find a valid WAL-E restore information',
detail='Expected to find a directory named .wal-e/restore_info'
)
logger.info('Verifying database against manifests from {0}'
''.format(manifests_dir))
with open(os.path.join(data_directory,
'.wal-e', 'restore-spec.json'), 'r') as f:
spec = json.load(f)
if self._database_verify(data_directory, manifests_dir, spec):
logger.info('Verification against manifests passed')
else:
logger.info('Verification against manifests FAILED')
raise UserException(
msg='Verification of database failed',
detail='Check logs for details of discrepancies found')
def database_fetch(self, pg_cluster_dir, backup_name,
blind_restore, restore_spec, pool_size):
if os.path.exists(os.path.join(pg_cluster_dir, 'postmaster.pid')):
hint = ('Shut down postgres. If there is a stale lockfile, '
'then remove it after being very sure postgres is not '
'running.')
raise UserException(
msg='attempting to overwrite a live data directory',
detail='Found a postmaster.pid lockfile, and aborting',
hint=hint)
bl = self._backup_list(False)
backups = list(bl.find_all(backup_name))
assert len(backups) <= 1
if len(backups) == 0:
raise UserException(
msg='no backups found for fetching',
detail=('No backup matching the query {0} '
'was able to be located.'.format(backup_name)))
elif len(backups) > 1:
raise UserException(
msg='more than one backup found for fetching',
detail=('More than one backup matching the query {0} was able '
'to be located.'.format(backup_name)),
hint='To list qualifying backups, '
'try "wal-e backup-list QUERY".')
# There must be exactly one qualifying backup at this point.
assert len(backups) == 1
assert backups[0] is not None
backup_info = backups[0]
backup_info.load_detail(self.new_connection())
self.layout.basebackup_tar_partition_directory(backup_info)
if restore_spec is not None:
if restore_spec != 'SOURCE':
if not os.path.isfile(restore_spec):
raise UserException(
msg='Restore specification does not exist',
detail='File not found: %s'.format(restore_spec),
hint=('Provide valid json-formatted restoration '
'specification, or pseudo-name "SOURCE" to '
'restore using the specification from the '
'backup progenitor.'))
with open(restore_spec, 'r') as fs:
spec = json.load(fs)
backup_info.spec.update(spec)
if 'base_prefix' not in backup_info.spec \
or not backup_info.spec['base_prefix']:
backup_info.spec['base_prefix'] = pg_cluster_dir
self._build_restore_paths(backup_info.spec)
else:
# If the user hasn't passed in a restoration specification
# use pg_cluster_dir as the resore prefix
backup_info.spec['base_prefix'] = pg_cluster_dir
manifests_dir = manifest.directory(pg_cluster_dir)
try:
os.makedirs(manifests_dir)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
if not blind_restore:
self._verify_restore_paths(backup_info.spec)
connections = []
for i in xrange(pool_size):
connections.append(self.new_connection())
partition_iter = self.worker.TarPartitionLister(
connections[0], self.layout, backup_info)
manifest_iter = self.worker.ManifestLister(
connections[0], self.layout, backup_info)
assert len(connections) == pool_size
fetchers = []
for i in xrange(pool_size):
fetchers.append(self.worker.BackupFetcher(
connections[i], self.layout, backup_info,
backup_info.spec['base_prefix'],
(self.gpg_key_id is not None)))
assert len(fetchers) == pool_size
p = gevent.pool.Pool(size=pool_size)
fetcher_cycle = itertools.cycle(fetchers)
for part_name in partition_iter:
p.spawn(
self._exception_gather_guard(
fetcher_cycle.next().fetch_partition),
part_name)
for part_name in manifest_iter:
p.spawn(
self._exception_gather_guard(
fetcher_cycle.next().fetch_manifest),
part_name, manifests_dir)
p.join(raise_error=True)
with open(os.path.join(pg_cluster_dir,
'.wal-e', 'restore-spec.json'), 'w') as f:
json.dump(backup_info.spec, f, indent=4)
with open(os.path.join(pg_cluster_dir,
'.wal-e', 'backup-info.json'), 'w') as f:
json.dump(backup_info.details(), f, indent=4)
logger.info('Verifying database against manifests from {0}'.
format(manifests_dir))
if self._database_verify(backup_info.spec['base_prefix'],
manifests_dir,
backup_info):
logger.info('Restore succeeded and passed verification')
else:
logger.info('Restore finished but FAILED verification')
raise UserException(
msg='Verification of database failed',
detail='Check logs for details of discrepancies found')
