def test_log_levels(monkeypatch):
l = log_help.WalELogger()
class DidLog(object):
def __init__(self):
self.called = False
def __call__(self, *args, **kwargs):
self.called = True
# Try the default logging level, which should log INFO-level.
d = DidLog()
monkeypatch.setattr(l._logger, 'log', d)
l.log(level=logging.INFO, msg='hello')
assert d.called is True
# Test default elision of DEBUG-level statements.
d = DidLog()
monkeypatch.setattr(l._logger, 'log', d)
l.log(level=logging.DEBUG, msg='hello')
assert d.called is False
# Adjust log level ignore INFO and below
log_help.MINIMUM_LOG_LEVEL = logging.WARNING
# Test elision of INFO level statements once minimum level has
# been adjusted.
d = DidLog()
monkeypatch.setattr(l._logger, 'log', d)
l.log(level=logging.INFO, msg='hello')
assert d.called is False
# Make sure WARNING level is still logged even if minimum level
# has been adjusted.
d = DidLog()
monkeypatch.setattr(l._logger, 'log', d)
l.log(level=logging.WARNING, msg='HELLO!')
assert d.called is True
import textwrap
import traceback
from wal_e import log_help
from wal_e.exception import UserCritical
from wal_e.exception import UserException
from wal_e import storage
from wal_e.piper import popen_sp
from wal_e.worker.pg import PSQL_BIN, psql_csv_run
from wal_e.pipeline import LZOP_BIN, PV_BIN, GPG_BIN
from wal_e.worker.pg import CONFIG_BIN, PgControlDataParser
log_help.configure(format='%(name)-12s %(levelname)-8s %(message)s')
logger = log_help.WalELogger('wal_e.main')
def external_program_check(to_check=frozenset([PSQL_BIN, LZOP_BIN, PV_BIN])):
"""
Validates the existence and basic working-ness of other programs
Implemented because it is easy to get confusing error output when
one does not install a dependency because of the fork-worker model
that is both necessary for throughput and makes more obscure the
cause of failures. This is intended to be a time and frustration
saving measure. This problem has confused The Author in practice
when switching rapidly between machines.
"""
import tempfile
import time
import boto.exception
from cStringIO import StringIO
from wal_e import log_help
from wal_e import pipebuf
from wal_e import pipeline
from wal_e import storage
from wal_e.blobstore import get_blobstore
from wal_e.piper import PIPE
from wal_e.retries import retry, retry_with_count
from wal_e.worker.worker_util import do_lzop_put, format_kib_per_second
logger = log_help.WalELogger(__name__)
class WalUploader(object):
def __init__(self, layout, creds, gpg_key_id):
self.layout = layout
self.creds = creds
self.gpg_key_id = gpg_key_id
self.blobstore = get_blobstore(layout)
def __call__(self, segment):
# TODO :: Move arbitray path construction to StorageLayout Object
url = '{0}/wal_{1}/{2}.lzo'.format(self.layout.prefix.rstrip('/'),
storage.CURRENT_VERSION,
segment.name)
import sys
import textwrap
import traceback
import wal_e.log_help as log_help
from wal_e.exception import UserException
from wal_e.operator import s3_operator
from wal_e.piper import popen_sp
from wal_e.worker.psql_worker import PSQL_BIN, psql_csv_run
from wal_e.pipeline import LZOP_BIN, PV_BIN, GPG_BIN
from wal_e.worker.pg_controldata_worker import CONFIG_BIN, PgControlDataParser
log_help.configure(format='%(name)-12s %(levelname)-8s %(message)s')
logger = log_help.WalELogger('wal_e.main', level=logging.INFO)
def external_program_check(to_check=frozenset([PSQL_BIN, LZOP_BIN, PV_BIN])):
"""
Validates the existence and basic working-ness of other programs
Implemented because it is easy to get confusing error output when
one does not install a dependency because of the fork-worker model
that is both necessary for throughput and makes more obscure the
cause of failures. This is intended to be a time and frustration
saving measure. This problem has confused The Author in practice
when switching rapidly between machines.
"""
from urlparse import urlparse
from boto.s3.connection import (S3Connection, SubdomainCallingFormat,
OrdinaryCallingFormat)
import wal_e.log_help as log_help
import wal_e.storage.s3_storage as s3_storage
from wal_e.exception import UserException
from wal_e.pipeline import (get_archive_upload_pipeline, get_download_pipeline,
get_upload_pipeline)
from wal_e.piper import PIPE
from wal_e.retries import retry, retry_with_count
from wal_e.worker import s3_deleter
logger = log_help.WalELogger(__name__, level=logging.INFO)
generic_weird_key_hint_message = ('This means an unexpected key was found in '
'a WAL-E prefix. It can be harmless, or '
'the result a bug or misconfiguration.')
# Set a timeout for boto HTTP operations should no timeout be set.
# Yes, in the case the user *wanted* no timeouts, this would set one.
# If that becomes a problem, someone should post a bug, although I am
# having a hard time imagining why that behavior could ever be useful.
if not boto.config.has_option('Boto', 'http_socket_timeout'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.set('Boto', 'http_socket_timeout', '5')
