def _reset_streaming_status(self, postgres_status, streaming_status):
"""
Reset the status of receive-wal by removing the .partial file that
is marking the current position and creating one that is current with
the PostgreSQL insert location
"""
current_wal = xlog.location_to_xlogfile_name_offset(
postgres_status["current_lsn"],
streaming_status["timeline"],
postgres_status["xlog_segment_size"],
)["file_name"]
restart_wal = current_wal
if (postgres_status["replication_slot"]
and postgres_status["replication_slot"].restart_lsn):
restart_wal = xlog.location_to_xlogfile_name_offset(
postgres_status["replication_slot"].restart_lsn,
streaming_status["timeline"],
postgres_status["xlog_segment_size"],
)["file_name"]
restart_path = os.path.join(self.config.streaming_wals_directory,
restart_wal)
restart_partial_path = restart_path + ".partial"
wal_files = sorted(glob(
os.path.join(self.config.streaming_wals_directory, "*")),
reverse=True)
# Pick the newer file
last = None
for last in wal_files:
if xlog.is_wal_file(last) or xlog.is_partial_file(last):
break
# Check if the status is already up-to-date
if not last or last == restart_partial_path or last == restart_path:
output.info("Nothing to do. Position of receive-wal is aligned.")
return
if os.path.basename(last) > current_wal:
output.error(
"The receive-wal position is ahead of PostgreSQL "
"current WAL lsn (%s > %s)",
os.path.basename(last),
postgres_status["current_xlog"],
)
return
output.info("Resetting receive-wal directory status")
if xlog.is_partial_file(last):
output.info("Removing status file %s" % last)
os.unlink(last)
output.info("Creating status file %s" % restart_partial_path)
open(restart_partial_path, "w").close()
def test_partial_file(self):
assert not xlog.is_partial_file("000000000000000200000001")
assert xlog.is_partial_file("00000001000000000000000A.partial")
assert xlog.is_partial_file("test/00000001000000000000000A.partial")
assert not xlog.is_partial_file("00000002.history")
assert not xlog.is_partial_file("00000000000000000000000.partial")
assert not xlog.is_partial_file("0000000000000000000000000.partial")
assert not xlog.is_partial_file("000000000000X00000000000.partial")
assert not xlog.is_partial_file("00000001000000000000000A.00000020.partial")
assert not xlog.is_any_xlog_file("test.00000001000000000000000A.partial")
assert not xlog.is_partial_file("00000001.partial")
def _truncate_partial_file_if_needed(self, xlog_segment_size):
"""
Truncate .partial WAL file if size is not 0 or xlog_segment_size
:param int xlog_segment_size:
"""
# Retrieve the partial list (only one is expected)
partial_files = glob(
os.path.join(self.config.streaming_wals_directory, "*.partial"))
# Take the last partial file, ignoring wrongly formatted file names
last_partial = None
for partial in partial_files:
if not is_partial_file(partial):
continue
if not last_partial or partial > last_partial:
last_partial = partial
# Skip further work if there is no good partial file
if not last_partial:
return
# If size is either 0 or wal_segment_size everything is fine...
partial_size = os.path.getsize(last_partial)
if partial_size == 0 or partial_size == xlog_segment_size:
return
# otherwise truncate the file to be empty. This is safe because
# pg_receivewal pads the file to the full size before start writing.
output.info("Truncating partial file %s that has wrong size %s "
"while %s was expected." %
(last_partial, partial_size, xlog_segment_size))
open(last_partial, "wb").close()
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.streaming_archiver_batch_size
# List and sort all files in the incoming directory
file_names = glob(
os.path.join(self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones and history files.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
# Ignore temporary files
if file_name.endswith('.tmp'):
continue
# If the file doesn't exist, it has been renamed/removed while
# we were reading the directory. Ignore it.
if not os.path.exists(file_name):
continue
if not os.path.isfile(file_name):
errors.append(file_name)
elif xlog.is_partial_file(file_name):
skip.append(file_name)
elif xlog.is_any_xlog_file(file_name):
files.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as normal files
if len(skip) > 1:
partials = skip[:-1]
_logger.info('Archiving partial files for server %s: %s' %
(self.config.name, ", ".join(
[os.path.basename(f) for f in partials])))
files.extend(partials)
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors,
skip=skip)
def test_partial_file(self):
assert not xlog.is_partial_file('000000000000000200000001')
assert xlog.is_partial_file('00000001000000000000000A.partial')
assert xlog.is_partial_file('test/00000001000000000000000A.partial')
assert not xlog.is_partial_file('00000002.history')
assert not xlog.is_partial_file('00000000000000000000000.partial')
assert not xlog.is_partial_file('0000000000000000000000000.partial')
assert not xlog.is_partial_file('000000000000X00000000000.partial')
assert not xlog.is_partial_file(
'00000001000000000000000A.00000020.partial')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.partial')
assert not xlog.is_partial_file('00000001.partial')
def test_partial_file(self):
assert not xlog.is_partial_file('000000000000000200000001')
assert xlog.is_partial_file('00000001000000000000000A.partial')
assert xlog.is_partial_file('test/00000001000000000000000A.partial')
assert not xlog.is_partial_file('00000002.history')
assert not xlog.is_partial_file('00000000000000000000000.partial')
assert not xlog.is_partial_file('0000000000000000000000000.partial')
assert not xlog.is_partial_file('000000000000X00000000000.partial')
assert not xlog.is_partial_file(
'00000001000000000000000A.00000020.partial')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.partial')
assert not xlog.is_partial_file('00000001.partial')
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverQueue: list of WAL files
"""
# Get the batch size from configuration (0 = unlimited)
batch_size = self.config.streaming_archiver_batch_size
# List and sort all files in the incoming directory
file_names = glob(os.path.join(
self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones and history files.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if not os.path.isfile(file_name):
errors.append(file_name)
elif xlog.is_partial_file(file_name):
skip.append(file_name)
elif xlog.is_any_xlog_file(file_name):
files.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
_logger.warning('Multiple partial files found for server %s: %s' %
(self.config.name,
", ".join([os.path.basename(f) for f in errors])))
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors, skip=skip)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverBatch: list of WAL files
"""
# List and sort all files in the incoming directory
file_names = glob(os.path.join(
self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if xlog.is_wal_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
elif xlog.is_partial_file(file_name) and os.path.isfile(file_name):
skip.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverBatch(wal_files, errors=errors, skip=skip)
def get_next_batch(self):
"""
Returns the next batch of WAL files that have been archived via
streaming replication (in the 'streaming' directory)
This method always leaves one file in the "streaming" directory,
because the 'pg_receivexlog' process needs at least one file to
detect the current streaming position after a restart.
:return: WalArchiverBatch: list of WAL files
"""
# List and sort all files in the incoming directory
file_names = glob(
os.path.join(self.config.streaming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file,
# including partial ones.
# Anything else is treated like an error/anomaly
files = []
skip = []
errors = []
for file_name in file_names:
if xlog.is_wal_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
elif xlog.is_partial_file(file_name) and os.path.isfile(file_name):
skip.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as errors
if len(skip) > 1:
errors.extend(skip[:-1])
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f, compression=None) for f in files]
return WalArchiverBatch(wal_files, errors=errors, skip=skip)