def test_non_dir_contents(self):
with temptree([]) as tmpdir:
data = os.path.join(tmpdir, "drive", "data")
os.makedirs(data)
with open(os.path.join(data, "partition1"), "w"):
pass
partition = os.path.join(data, "partition2")
os.makedirs(partition)
with open(os.path.join(partition, "suffix1"), "w"):
pass
suffix = os.path.join(partition, "suffix2")
os.makedirs(suffix)
with open(os.path.join(suffix, "hash1"), "w"):
pass
locations = ondisk.audit_location_generator(
tmpdir, "data", mount_check=False
)
self.assertEqual(list(locations), [])
def test_search_tree(self):
# file match & ext miss
with temptree(["asdf.conf", "blarg.conf", "asdf.cfg"]) as t:
asdf = utils.search_tree(t, "a*", ".conf")
self.assertEquals(len(asdf), 1)
self.assertEquals(asdf[0], os.path.join(t, "asdf.conf"))
# multi-file match & glob miss & sort
with temptree(["application.bin", "apple.bin", "apropos.bin"]) as t:
app_bins = utils.search_tree(t, "app*", "bin")
self.assertEquals(len(app_bins), 2)
self.assertEquals(app_bins[0], os.path.join(t, "apple.bin"))
self.assertEquals(app_bins[1], os.path.join(t, "application.bin"))
# test file in folder & ext miss & glob miss
files = ("sub/file1.ini", "sub/file2.conf", "sub.bin", "bus.ini", "bus/file3.ini")
with temptree(files) as t:
sub_ini = utils.search_tree(t, "sub*", ".ini")
self.assertEquals(len(sub_ini), 1)
self.assertEquals(sub_ini[0], os.path.join(t, "sub/file1.ini"))
# test multi-file in folder & sub-folder & ext miss & glob miss
files = ("folder_file.txt", "folder/1.txt", "folder/sub/2.txt", "folder2/3.txt", "Folder3/4.txt" "folder.rc")
with temptree(files) as t:
folder_texts = utils.search_tree(t, "folder*", ".txt")
self.assertEquals(len(folder_texts), 4)
f1 = os.path.join(t, "folder_file.txt")
f2 = os.path.join(t, "folder/1.txt")
f3 = os.path.join(t, "folder/sub/2.txt")
f4 = os.path.join(t, "folder2/3.txt")
for f in [f1, f2, f3, f4]:
self.assert_(f in folder_texts)
def test_pattern_upload_all_logs(self):
# test empty dir
with temptree([]) as t:
conf = {'log_dir': t}
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.run_once)
def get_random_length_str(max_len=10, chars=string.ascii_letters):
return ''.join(random.choice(chars) for x in
range(random.randint(1, max_len)))
template = 'prefix_%(random)s_%(digits)s.blah.' \
'%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz'
pattern = '''prefix_.*_[0-9]+\.blah\.
(?P<year>[0-9]{4})
(?P<month>[0-1][0-9])
(?P<day>[0-3][0-9])
(?P<hour>[0-2][0-9])00-[0-9]{2}00
-[0-9]?[0-9]\.gz'''
files_that_should_match = []
# add some files that match
for i in range(24):
fname = template % {
'random': get_random_length_str(),
'digits': get_random_length_str(16, string.digits),
'datestr': datetime.now().strftime('%Y%m%d'),
'hour': i,
'next_hour': i + 1,
'number': random.randint(0, 20),
}
files_that_should_match.append(fname)
# add some files that don't match
files = list(files_that_should_match)
for i in range(24):
fname = template % {
'random': get_random_length_str(),
'digits': get_random_length_str(16, string.digits),
'datestr': datetime.now().strftime('%Y%m'),
'hour': i,
'next_hour': i + 1,
'number': random.randint(0, 20),
}
files.append(fname)
for fname in files:
print fname
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
self.assertEquals(len(os.listdir(t)), 48)
conf = {'source_filename_pattern': pattern, 'log_dir': t}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(os.listdir(t)), 24)
self.assertEquals(len(uploader.uploaded_files), 24)
files_that_were_uploaded = set(x[0] for x in
uploader.uploaded_files)
for f in files_that_should_match:
self.assert_(os.path.join(t, f) in files_that_were_uploaded)
def test_pattern_upload_all_logs(self):
# test empty dir
with temptree([]) as t:
conf = {'log_dir': t}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
def get_random_length_str(max_len=10, chars=string.ascii_letters):
return ''.join(
random.choice(chars)
for x in range(random.randint(1, max_len)))
template = 'prefix_%(random)s_%(digits)s.blah.' \
'%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz'
pattern = r'prefix_.*_[0-9]+\.blah\.%Y%m%d%H00-[0-9]{2}00' \
'-[0-9]?[0-9]\.gz'
files_that_should_match = []
# add some files that match
for i in range(24):
fname = template % {
'random': get_random_length_str(),
'digits': get_random_length_str(16, string.digits),
'datestr': datetime.now().strftime('%Y%m%d'),
'hour': i,
'next_hour': i + 1,
'number': random.randint(0, 20),
}
files_that_should_match.append(fname)
# add some files that don't match
files = list(files_that_should_match)
for i in range(24):
fname = template % {
'random': get_random_length_str(),
'digits': get_random_length_str(16, string.digits),
'datestr': datetime.now().strftime('%Y%m'),
'hour': i,
'next_hour': i + 1,
'number': random.randint(0, 20),
}
files.append(fname)
for fname in files:
print fname
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
self.assertEquals(len(os.listdir(t)), 48)
conf = {'source_filename_pattern': pattern, 'log_dir': t}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(os.listdir(t)), 24)
self.assertEquals(len(uploader.uploaded_files), 24)
files_that_were_uploaded = set(x[0]
for x in uploader.uploaded_files)
for f in files_that_should_match:
self.assert_(os.path.join(t, f) in files_that_were_uploaded)
def test_search_tree(self):
# file match & ext miss
with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t:
asdf = utils.search_tree(t, 'a*', '.conf')
self.assertEquals(len(asdf), 1)
self.assertEquals(asdf[0],
os.path.join(t, 'asdf.conf'))
# multi-file match & glob miss & sort
with temptree(['application.bin', 'apple.bin', 'apropos.bin']) as t:
app_bins = utils.search_tree(t, 'app*', 'bin')
self.assertEquals(len(app_bins), 2)
self.assertEquals(app_bins[0],
os.path.join(t, 'apple.bin'))
self.assertEquals(app_bins[1],
os.path.join(t, 'application.bin'))
# test file in folder & ext miss & glob miss
files = (
'sub/file1.ini',
'sub/file2.conf',
'sub.bin',
'bus.ini',
'bus/file3.ini',
)
with temptree(files) as t:
sub_ini = utils.search_tree(t, 'sub*', '.ini')
self.assertEquals(len(sub_ini), 1)
self.assertEquals(sub_ini[0],
os.path.join(t, 'sub/file1.ini'))
# test multi-file in folder & sub-folder & ext miss & glob miss
files = (
'folder_file.txt',
'folder/1.txt',
'folder/sub/2.txt',
'folder2/3.txt',
'Folder3/4.txt'
'folder.rc',
)
with temptree(files) as t:
folder_texts = utils.search_tree(t, 'folder*', '.txt')
self.assertEquals(len(folder_texts), 4)
f1 = os.path.join(t, 'folder_file.txt')
f2 = os.path.join(t, 'folder/1.txt')
f3 = os.path.join(t, 'folder/sub/2.txt')
f4 = os.path.join(t, 'folder2/3.txt')
for f in [f1, f2, f3, f4]:
self.assert_(f in folder_texts)
def test_search_tree(self):
# file match & ext miss
with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t:
asdf = utils.search_tree(t, 'a*', '.conf')
self.assertEquals(len(asdf), 1)
self.assertEquals(asdf[0],
os.path.join(t, 'asdf.conf'))
# multi-file match & glob miss & sort
with temptree(['application.bin', 'apple.bin', 'apropos.bin']) as t:
app_bins = utils.search_tree(t, 'app*', 'bin')
self.assertEquals(len(app_bins), 2)
self.assertEquals(app_bins[0],
os.path.join(t, 'apple.bin'))
self.assertEquals(app_bins[1],
os.path.join(t, 'application.bin'))
# test file in folder & ext miss & glob miss
files = (
'sub/file1.ini',
'sub/file2.conf',
'sub.bin',
'bus.ini',
'bus/file3.ini',
)
with temptree(files) as t:
sub_ini = utils.search_tree(t, 'sub*', '.ini')
self.assertEquals(len(sub_ini), 1)
self.assertEquals(sub_ini[0],
os.path.join(t, 'sub/file1.ini'))
# test multi-file in folder & sub-folder & ext miss & glob miss
files = (
'folder_file.txt',
'folder/1.txt',
'folder/sub/2.txt',
'folder2/3.txt',
'Folder3/4.txt'
'folder.rc',
)
with temptree(files) as t:
folder_texts = utils.search_tree(t, 'folder*', '.txt')
self.assertEquals(len(folder_texts), 4)
f1 = os.path.join(t, 'folder_file.txt')
f2 = os.path.join(t, 'folder/1.txt')
f3 = os.path.join(t, 'folder/sub/2.txt')
f4 = os.path.join(t, 'folder2/3.txt')
for f in [f1, f2, f3, f4]:
self.assert_(f in folder_texts)
def test_create_container_fail(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files) as t:
conf = {'log_dir': t}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
with temptree(files) as t:
conf = {'log_dir': t}
uploader = MockLogUploader(conf)
# mock create_container to fail
uploader.internal_proxy.create_container = lambda *args: False
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
def test_create_container_fail(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files) as t:
conf = {'log_dir': t}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
with temptree(files) as t:
conf = {'log_dir': t}
uploader = MockLogUploader(conf)
# mock create_container to fail
uploader.internal_proxy.create_container = lambda *args: False
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
def test_two_realms_and_change_a_default(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[DEFAULT]
mtime_check_interval = 60
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
[UK]
key = e9569809dc8b4951accc1487aa788012
key2 = f6351bd1cc36413baa43f7ba1b45e51d
cluster_lon3 = http://lon3.host/v1/
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 60)
self.assertEqual(sorted(csr.realms()), ['UK', 'US'])
self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b')
self.assertIsNone(csr.key2('US'))
self.assertEqual(csr.clusters('US'), ['DFW1'])
self.assertEqual(csr.endpoint('US', 'DFW1'),
'http://dfw1.host/v1/')
self.assertEqual(csr.key('UK'), 'e9569809dc8b4951accc1487aa788012')
self.assertEqual(csr.key2('UK'),
'f6351bd1cc36413baa43f7ba1b45e51d')
self.assertEqual(csr.clusters('UK'), ['LON3'])
self.assertEqual(csr.endpoint('UK', 'LON3'),
'http://lon3.host/v1/')
def _proxy_modify_wsgi_pipeline(self, pipe):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = %s
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:gatekeeper]
use = egg:swift#gatekeeper
"""
config = config % (pipe,)
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
app = wsgi.loadapp(conf_file, global_conf={})
return app
def test_appconfig_dir_ignores_hidden_files(self):
config_dir = {
'server.conf.d/01.conf':
"""
[app:main]
use = egg:swift#proxy
port = 8080
""",
'server.conf.d/.01.conf.swp':
"""
[app:main]
use = egg:swift#proxy
port = 8081
""",
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with temptree(*zip(*config_dir.items())) as path:
conf_dir = os.path.join(path, 'server.conf.d')
conf = wsgi.appconfig(conf_dir)
expected = {
'__file__': os.path.join(path, 'server.conf.d'),
'here': os.path.join(path, 'server.conf.d'),
'port': '8080',
}
self.assertEquals(conf, expected)
def test_init_request_processor(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = catch_errors proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:catch_errors]
use = egg:swift#catch_errors
"""
contents = dedent(config)
with temptree(["proxy-server.conf"]) as t:
conf_file = os.path.join(t, "proxy-server.conf")
with open(conf_file, "w") as f:
f.write(contents.replace("TEMPDIR", t))
_fake_rings(t)
app, conf, logger, log_name = wsgi.init_request_processor(conf_file, "proxy-server")
# verify pipeline is catch_errors -> proxy-servery
expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assert_(isinstance(app, expected))
self.assert_(isinstance(app.app, swift.proxy.server.Application))
# config settings applied to app instance
self.assertEquals(0.2, app.app.conn_timeout)
# appconfig returns values from 'proxy-server' section
expected = {"__file__": conf_file, "here": os.path.dirname(conf_file), "conn_timeout": "0.2", "swift_dir": t}
self.assertEquals(expected, conf)
# logger works
logger.info("testing")
self.assertEquals("proxy-server", log_name)
def test_two_realms_and_change_a_default(self):
fname = "container-sync-realms.conf"
fcontents = """
[DEFAULT]
mtime_check_interval = 60
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
[UK]
key = e9569809dc8b4951accc1487aa788012
key2 = f6351bd1cc36413baa43f7ba1b45e51d
cluster_lon3 = http://lon3.host/v1/
"""
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.lines_dict, {})
self.assertEqual(csr.mtime_check_interval, 60)
self.assertEqual(sorted(csr.realms()), ["UK", "US"])
self.assertEqual(csr.key("US"), "9ff3b71c849749dbaec4ccdd3cbab62b")
self.assertEqual(csr.key2("US"), None)
self.assertEqual(csr.clusters("US"), ["DFW1"])
self.assertEqual(csr.endpoint("US", "DFW1"), "http://dfw1.host/v1/")
self.assertEqual(csr.key("UK"), "e9569809dc8b4951accc1487aa788012")
self.assertEqual(csr.key2("UK"), "f6351bd1cc36413baa43f7ba1b45e51d")
self.assertEqual(csr.clusters("UK"), ["LON3"])
self.assertEqual(csr.endpoint("UK", "LON3"), "http://lon3.host/v1/")
def test_two_realms_and_change_a_default(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[DEFAULT]
mtime_check_interval = 60
[US]
key = 9ff3b71c849749dbaec4ccdd3cbab62b
cluster_dfw1 = http://dfw1.host/v1/
[UK]
key = e9569809dc8b4951accc1487aa788012
key2 = f6351bd1cc36413baa43f7ba1b45e51d
cluster_lon3 = http://lon3.host/v1/
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 60)
self.assertEqual(sorted(csr.realms()), ['UK', 'US'])
self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b')
self.assertEqual(csr.key2('US'), None)
self.assertEqual(csr.clusters('US'), ['DFW1'])
self.assertEqual(
csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/')
self.assertEqual(csr.key('UK'), 'e9569809dc8b4951accc1487aa788012')
self.assertEqual(
csr.key2('UK'), 'f6351bd1cc36413baa43f7ba1b45e51d')
self.assertEqual(csr.clusters('UK'), ['LON3'])
self.assertEqual(
csr.endpoint('UK', 'LON3'), 'http://lon3.host/v1/')
def setUp(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = healthcheck catch_errors tempurl proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:tempurl]
paste.filter_factory = swift.common.middleware.tempurl:filter_factory
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={})
self.pipe = wsgi.PipelineWrapper(ctx)
def test_proxy_modify_wsgi_pipeline_ordering(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = healthcheck proxy-logging bulk tempurl proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:proxy-logging]
use = egg:swift#proxy_logging
[filter:bulk]
use = egg:swift#bulk
[filter:tempurl]
use = egg:swift#tempurl
"""
new_req_filters = [
# not in pipeline, no afters
{
'name': 'catch_errors'
},
# already in pipeline
{
'name': 'proxy_logging',
'after_fn': lambda _: ['catch_errors']
},
# not in pipeline, comes after more than one thing
{
'name': 'container_quotas',
'after_fn': lambda _: ['catch_errors', 'bulk']
}
]
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
with mock.patch.object(swift.proxy.server, 'required_filters',
new_req_filters):
app = wsgi.loadapp(conf_file, global_conf={})
self.assertEqual(self.pipeline_modules(app), [
'swift.common.middleware.catch_errors',
'swift.common.middleware.healthcheck',
'swift.common.middleware.proxy_logging',
'swift.common.middleware.bulk',
'swift.common.middleware.container_quotas',
'swift.common.middleware.tempurl', 'swift.proxy.server'
])
def test_proxy_unmodified_wsgi_pipeline(self):
# Make sure things are sane even when we modify nothing
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = catch_errors gatekeeper proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:gatekeeper]
use = egg:swift#gatekeeper
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
app = wsgi.loadapp(conf_file, global_conf={})
self.assertEqual(self.pipeline_modules(app),
['swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.proxy.server'])
def test_run_server_with_latest_eventlet(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = proxy-server
[app:proxy-server]
use = egg:swift#proxy
"""
def argspec_stub(server):
return mock.MagicMock(args=["capitalize_response_headers"])
contents = dedent(config)
with temptree(["proxy-server.conf"]) as t:
conf_file = os.path.join(t, "proxy-server.conf")
with open(conf_file, "w") as f:
f.write(contents.replace("TEMPDIR", t))
_fake_rings(t)
with nested(
mock.patch("swift.proxy.server.Application." "modify_wsgi_pipeline"),
mock.patch("swift.common.wsgi.wsgi"),
mock.patch("swift.common.wsgi.eventlet"),
mock.patch("swift.common.wsgi.inspect", getargspec=argspec_stub),
) as (_, _wsgi, _, _):
conf = wsgi.appconfig(conf_file)
logger = logging.getLogger("test")
sock = listen(("localhost", 0))
wsgi.run_server(conf, logger, sock)
_wsgi.server.assert_called()
args, kwargs = _wsgi.server.call_args
self.assertEquals(kwargs.get("capitalize_response_headers"), False)
def test_proxy_modify_wsgi_pipeline(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = healthcheck proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:healthcheck]
use = egg:swift#healthcheck
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
app = wsgi.loadapp(conf_file, global_conf={})
self.assertEqual(self.pipeline_modules(app),
['swift.common.middleware.catch_errors',
'swift.common.middleware.gatekeeper',
'swift.common.middleware.healthcheck',
'swift.proxy.server'])
def test_bad_pattern_in_config(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
# invalid pattern
conf = {
'log_dir': t,
'source_filename_pattern': '%Y%m%d%h'
} # should be %H
uploader = MockLogUploader(conf)
self.assertFalse(uploader.validate_filename_pattern())
uploader.upload_all_logs()
self.assertEquals(uploader.uploaded_files, [])
conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%H'}
uploader = MockLogUploader(conf)
self.assert_(uploader.validate_filename_pattern())
uploader.upload_all_logs()
self.assertEquals(len(uploader.uploaded_files), 1)
# deprecated warning on source_filename_format
class MockLogger():
def __init__(self):
self.msgs = defaultdict(list)
def log(self, level, msg):
self.msgs[level].append(msg)
def __getattr__(self, attr):
return partial(self.log, attr)
logger = MockLogger.logger = MockLogger()
def mock_get_logger(*args, **kwargs):
return MockLogger.logger
_orig_get_logger = log_uploader.utils.get_logger
try:
log_uploader.utils.get_logger = mock_get_logger
conf = {'source_filename_format': '%Y%m%d%H'}
uploader = MockLogUploader(conf, logger=logger)
self.assert_(
[m for m in logger.msgs['warning'] if 'deprecated' in m])
finally:
log_uploader.utils.get_logger = _orig_get_logger
# convert source_filename_format to regex
conf = {'source_filename_format': 'pattern-*.%Y%m%d%H.*.gz'}
uploader = MockLogUploader(conf)
expected = r'pattern-.*\.%Y%m%d%H\..*\.gz'
self.assertEquals(uploader.pattern, expected)
# use source_filename_pattern if we have the choice!
conf = {
'source_filename_format': 'bad',
'source_filename_pattern': 'good',
}
uploader = MockLogUploader(conf)
self.assertEquals(uploader.pattern, 'good')
def test_bad_pattern_in_config(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
# invalid pattern
conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%h'} # should be %H
uploader = MockLogUploader(conf)
self.assertFalse(uploader.validate_filename_pattern())
uploader.upload_all_logs()
self.assertEquals(uploader.uploaded_files, [])
conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%H'}
uploader = MockLogUploader(conf)
self.assert_(uploader.validate_filename_pattern())
uploader.upload_all_logs()
self.assertEquals(len(uploader.uploaded_files), 1)
# deprecated warning on source_filename_format
class MockLogger():
def __init__(self):
self.msgs = defaultdict(list)
def log(self, level, msg):
self.msgs[level].append(msg)
def __getattr__(self, attr):
return partial(self.log, attr)
logger = MockLogger.logger = MockLogger()
def mock_get_logger(*args, **kwargs):
return MockLogger.logger
_orig_get_logger = log_uploader.utils.get_logger
try:
log_uploader.utils.get_logger = mock_get_logger
conf = {'source_filename_format': '%Y%m%d%H'}
uploader = MockLogUploader(conf, logger=logger)
self.assert_([m for m in logger.msgs['warning']
if 'deprecated' in m])
finally:
log_uploader.utils.get_logger = _orig_get_logger
# convert source_filename_format to regex
conf = {'source_filename_format': 'pattern-*.%Y%m%d%H.*.gz'}
uploader = MockLogUploader(conf)
expected = r'pattern-.*\.%Y%m%d%H\..*\.gz'
self.assertEquals(uploader.pattern, expected)
# use source_filename_pattern if we have the choice!
conf = {
'source_filename_format': 'bad',
'source_filename_pattern': 'good',
}
uploader = MockLogUploader(conf)
self.assertEquals(uploader.pattern, 'good')
def test_load_app(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = healthcheck proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:healthcheck]
use = egg:swift#healthcheck
"""
def modify_func(app, pipe):
new = pipe.create_filter('catch_errors')
pipe.insert_filter(new)
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
with mock.patch(
'swift.proxy.server.Application.modify_wsgi_pipeline',
modify_func):
app = wsgi.loadapp(conf_file, global_conf={})
exp = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assertTrue(isinstance(app, exp), app)
exp = swift.common.middleware.healthcheck.HealthCheckMiddleware
self.assertTrue(isinstance(app.app, exp), app.app)
exp = swift.proxy.server.Application
self.assertTrue(isinstance(app.app.app, exp), app.app.app)
# make sure you can turn off the pipeline modification if you want
def blow_up(*_, **__):
raise self.fail("needs more struts")
with mock.patch(
'swift.proxy.server.Application.modify_wsgi_pipeline',
blow_up):
app = wsgi.loadapp(conf_file,
global_conf={},
allow_modify_pipeline=False)
# the pipeline was untouched
exp = swift.common.middleware.healthcheck.HealthCheckMiddleware
self.assertTrue(isinstance(app, exp), app)
exp = swift.proxy.server.Application
self.assertTrue(isinstance(app.app, exp), app.app)
def test_empty(self):
fname = "container-sync-realms.conf"
fcontents = ""
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.lines_dict, {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), [])
def test_empty(self):
fname = 'container-sync-realms.conf'
fcontents = ''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), [])
def test_proxy_modify_wsgi_pipeline_ordering(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = healthcheck proxy-logging bulk tempurl proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:proxy-logging]
use = egg:swift#proxy_logging
[filter:bulk]
use = egg:swift#bulk
[filter:tempurl]
use = egg:swift#tempurl
"""
new_req_filters = [
# not in pipeline, no afters
{"name": "catch_errors"},
# already in pipeline
{"name": "proxy_logging", "after_fn": lambda _: ["catch_errors"]},
# not in pipeline, comes after more than one thing
{"name": "container_quotas", "after_fn": lambda _: ["catch_errors", "bulk"]},
]
contents = dedent(config)
with temptree(["proxy-server.conf"]) as t:
conf_file = os.path.join(t, "proxy-server.conf")
with open(conf_file, "w") as f:
f.write(contents.replace("TEMPDIR", t))
_fake_rings(t)
with mock.patch.object(swift.proxy.server, "required_filters", new_req_filters):
app = wsgi.loadapp(conf_file, global_conf={})
self.assertEqual(
self.pipeline_modules(app),
[
"swift.common.middleware.catch_errors",
"swift.common.middleware.healthcheck",
"swift.common.middleware.proxy_logging",
"swift.common.middleware.bulk",
"swift.common.middleware.container_quotas",
"swift.common.middleware.tempurl",
"swift.proxy.server",
],
)
def test_load_app(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = healthcheck proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:healthcheck]
use = egg:swift#healthcheck
"""
def modify_func(app, pipe):
new = pipe.create_filter('catch_errors')
pipe.insert_filter(new)
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
with mock.patch(
'swift.proxy.server.Application.modify_wsgi_pipeline',
modify_func):
app = wsgi.loadapp(conf_file, global_conf={})
exp = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assertTrue(isinstance(app, exp), app)
exp = swift.common.middleware.healthcheck.HealthCheckMiddleware
self.assertTrue(isinstance(app.app, exp), app.app)
exp = swift.proxy.server.Application
self.assertTrue(isinstance(app.app.app, exp), app.app.app)
# make sure you can turn off the pipeline modification if you want
def blow_up(*_, **__):
raise self.fail("needs more struts")
with mock.patch(
'swift.proxy.server.Application.modify_wsgi_pipeline',
blow_up):
app = wsgi.loadapp(conf_file, global_conf={},
allow_modify_pipeline=False)
# the pipeline was untouched
exp = swift.common.middleware.healthcheck.HealthCheckMiddleware
self.assertTrue(isinstance(app, exp), app)
exp = swift.proxy.server.Application
self.assertTrue(isinstance(app.app, exp), app.app)
def test_log_cutoff(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files) as t:
conf = {'log_dir': t, 'new_log_cutoff': '7200'}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
conf = {'log_dir': t, 'new_log_cutoff': '0'}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
def test_log_cutoff(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files) as t:
conf = {'log_dir': t, 'new_log_cutoff': '7200'}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
conf = {'log_dir': t, 'new_log_cutoff': '0'}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
def test_remove_file(self):
with temptree([]) as t:
file_name = os.path.join(t, 'blah.pid')
# assert no raise
self.assertEquals(os.path.exists(file_name), False)
self.assertEquals(utils.remove_file(file_name), None)
with open(file_name, 'w') as f:
f.write('1')
self.assert_(os.path.exists(file_name))
self.assertEquals(utils.remove_file(file_name), None)
self.assertFalse(os.path.exists(file_name))
def test_get_sig(self):
fname = 'container-sync-realms.conf'
fcontents = ''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(
csr.get_sig('GET', '/some/path', '1387212345.67890',
'my_nonce', 'realm_key', 'user_key'),
'5a6eb486eb7b44ae1b1f014187a94529c3f9c8f9')
def test_remove_file(self):
with temptree([]) as t:
file_name = os.path.join(t, 'blah.pid')
# assert no raise
self.assertEquals(os.path.exists(file_name), False)
self.assertEquals(utils.remove_file(file_name), None)
with open(file_name, 'w') as f:
f.write('1')
self.assert_(os.path.exists(file_name))
self.assertEquals(utils.remove_file(file_name), None)
self.assertFalse(os.path.exists(file_name))
def test_get_sig(self):
fname = "container-sync-realms.conf"
fcontents = ""
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(
csr.get_sig("GET", "/some/path", "1387212345.67890", "my_nonce", "realm_key", "user_key"),
"5a6eb486eb7b44ae1b1f014187a94529c3f9c8f9",
)
def temp_config(contents):
fd, fn = mkstemp()
with temptree(['proxy-server.conf']) as t:
conf_dir = os.path.join(t, 'proxy-server.conf.d')
os.mkdir(conf_dir)
conf_file = os.path.join(conf_dir, 'default.conf')
with open(conf_file, 'w') as f:
f.write(contents)
f.close()
loader = wsgi.ConfigDirLoader(conf_dir)
yield loader.parser
def test_proxy_modify_wsgi_pipeline_ordering(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = healthcheck proxy-logging bulk tempurl proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:proxy-logging]
use = egg:swift#proxy_logging
[filter:bulk]
use = egg:swift#bulk
[filter:tempurl]
use = egg:swift#tempurl
"""
new_req_filters = [
# not in pipeline, no afters
{'name': 'catch_errors'},
# already in pipeline
{'name': 'proxy_logging',
'after': ['catch_errors']},
# not in pipeline, comes after more than one thing
{'name': 'container_quotas',
'after': ['catch_errors', 'bulk']}]
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
with mock.patch.object(swift.proxy.server, 'required_filters',
new_req_filters):
app = wsgi.loadapp(conf_file, global_conf={})
self.assertEqual(self.pipeline_modules(app), [
'swift.common.middleware.catch_errors',
'swift.common.middleware.healthcheck',
'swift.common.middleware.proxy_logging',
'swift.common.middleware.bulk',
'swift.common.middleware.container_quotas',
'swift.common.middleware.tempurl',
'swift.proxy.server'])
def test_get_sig(self):
fname = 'container-sync-realms.conf'
fcontents = ''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(
csr.get_sig(
'GET', '/some/path', '1387212345.67890', 'my_nonce',
'realm_key', 'user_key'),
'5a6eb486eb7b44ae1b1f014187a94529c3f9c8f9')
def test_run_server_conf_dir(self):
config_dir = {
'proxy-server.conf.d/pipeline.conf':
"""
[pipeline:main]
pipeline = proxy-server
""",
'proxy-server.conf.d/app.conf':
"""
[app:proxy-server]
use = egg:swift#proxy
""",
'proxy-server.conf.d/default.conf':
"""
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
"""
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with temptree(*zip(*config_dir.items())) as conf_root:
conf_dir = os.path.join(conf_root, 'proxy-server.conf.d')
with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f:
f.write('[DEFAULT]\nswift_dir = %s' % conf_root)
_fake_rings(conf_root)
with mock.patch('swift.proxy.server.Application.'
'modify_wsgi_pipeline'):
with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
with mock.patch.dict('os.environ', {'TZ': ''}):
with mock.patch('swift.common.wsgi.inspect'):
conf = wsgi.appconfig(conf_dir)
logger = logging.getLogger('test')
sock = listen(('localhost', 0))
wsgi.run_server(conf, logger, sock)
self.assert_(os.environ['TZ'] is not '')
self.assertEquals('HTTP/1.0',
_wsgi.HttpProtocol.default_request_version)
self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
_wsgi.server.assert_called()
args, kwargs = _wsgi.server.call_args
server_sock, server_app, server_logger = args
self.assertEquals(sock, server_sock)
self.assert_(isinstance(server_app, swift.proxy.server.Application))
self.assert_(isinstance(server_logger, wsgi.NullLogger))
self.assert_('custom_pool' in kwargs)
def test_run_server_debug(self):
config = """
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
max_clients = 1000
swift_dir = TEMPDIR
[pipeline:main]
pipeline = proxy-server
[app:proxy-server]
use = egg:swift#proxy
# while "set" values normally override default
set client_timeout = 20
# this section is not in conf during run_server
set max_clients = 10
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
with mock.patch('swift.proxy.server.Application.'
'modify_wsgi_pipeline'):
with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
mock_server = _wsgi.server
_wsgi.server = lambda *args, **kwargs: mock_server(
*args, **kwargs)
with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
conf = wsgi.appconfig(conf_file)
logger = logging.getLogger('test')
sock = listen(('localhost', 0))
wsgi.run_server(conf, logger, sock)
self.assertEquals('HTTP/1.0',
_wsgi.HttpProtocol.default_request_version)
self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
self.assertTrue(mock_server.called)
args, kwargs = mock_server.call_args
server_sock, server_app, server_logger = args
self.assertEquals(sock, server_sock)
self.assert_(isinstance(server_app, swift.proxy.server.Application))
self.assertEquals(20, server_app.client_timeout)
self.assertEqual(server_logger, None)
self.assert_('custom_pool' in kwargs)
self.assertEquals(1000, kwargs['custom_pool'].size)
def test_run_server_debug(self):
config = """
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
max_clients = 1000
swift_dir = TEMPDIR
[pipeline:main]
pipeline = proxy-server
[app:proxy-server]
use = egg:swift#proxy
# while "set" values normally override default
set client_timeout = 20
# this section is not in conf during run_server
set max_clients = 10
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
with mock.patch('swift.proxy.server.Application.'
'modify_wsgi_pipeline'):
with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
mock_server = _wsgi.server
_wsgi.server = lambda *args, **kwargs: mock_server(
*args, **kwargs)
with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
conf = wsgi.appconfig(conf_file)
logger = logging.getLogger('test')
sock = listen(('localhost', 0))
wsgi.run_server(conf, logger, sock)
self.assertEquals('HTTP/1.0',
_wsgi.HttpProtocol.default_request_version)
self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
self.assertTrue(mock_server.called)
args, kwargs = mock_server.call_args
server_sock, server_app, server_logger = args
self.assertEquals(sock, server_sock)
self.assert_(isinstance(server_app, swift.proxy.server.Application))
self.assertEquals(20, server_app.client_timeout)
self.assertEqual(server_logger, None)
self.assert_('custom_pool' in kwargs)
self.assertEquals(1000, kwargs['custom_pool'].size)
def test_bad_upload(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
# invalid pattern
conf = {'log_dir': t,
'source_filename_pattern': '%Y%m%d%h'} # should be %H
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.upload_all_logs)
conf = {'log_dir': t, 'source_filename_pattern': access_regex}
uploader = ErrorLogUploader(conf)
# this tests if the exception is handled
uploader.upload_all_logs()
def test_bad_pattern_in_config(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
# invalid pattern
conf = {'log_dir': t,
'source_filename_pattern': '%Y%m%d%h'} # should be %H
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.upload_all_logs)
conf = {'log_dir': t, 'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
uploader.upload_all_logs()
self.assertEquals(len(uploader.uploaded_files), 1)
def test_run_server_conf_dir(self):
config_dir = {
'proxy-server.conf.d/pipeline.conf': """
[pipeline:main]
pipeline = proxy-server
""",
'proxy-server.conf.d/app.conf': """
[app:proxy-server]
use = egg:swift#proxy
""",
'proxy-server.conf.d/default.conf': """
[DEFAULT]
eventlet_debug = yes
client_timeout = 30
"""
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with temptree(*zip(*config_dir.items())) as conf_root:
conf_dir = os.path.join(conf_root, 'proxy-server.conf.d')
with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f:
f.write('[DEFAULT]\nswift_dir = %s' % conf_root)
_fake_rings(conf_root)
with mock.patch('swift.proxy.server.Application.'
'modify_wsgi_pipeline'):
with mock.patch('swift.common.wsgi.wsgi') as _wsgi:
with mock.patch('swift.common.wsgi.eventlet') as _eventlet:
with mock.patch.dict('os.environ', {'TZ': ''}):
with mock.patch('swift.common.wsgi.inspect'):
conf = wsgi.appconfig(conf_dir)
logger = logging.getLogger('test')
sock = listen(('localhost', 0))
wsgi.run_server(conf, logger, sock)
self.assert_(os.environ['TZ'] is not '')
self.assertEquals('HTTP/1.0',
_wsgi.HttpProtocol.default_request_version)
self.assertEquals(30, _wsgi.WRITE_TIMEOUT)
_eventlet.hubs.use_hub.assert_called_with(utils.get_hub())
_eventlet.patcher.monkey_patch.assert_called_with(all=False,
socket=True)
_eventlet.debug.hub_exceptions.assert_called_with(True)
_wsgi.server.assert_called()
args, kwargs = _wsgi.server.call_args
server_sock, server_app, server_logger = args
self.assertEquals(sock, server_sock)
self.assert_(isinstance(server_app, swift.proxy.server.Application))
self.assert_(isinstance(server_logger, wsgi.NullLogger))
self.assert_('custom_pool' in kwargs)
def test_error_parsing(self):
fname = 'container-sync-realms.conf'
fcontents = 'invalid'
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(
logger.lines_dict,
{'error': [
"Could not load '%s': File contains no section headers.\n"
"file: %s, line: 1\n"
"'invalid'" % (fpath, fpath)]})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), [])
def test_upload_file_failed(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA]) as t:
conf = {'log_dir': t, 'unlink_log': 'true'}
uploader = MockLogUploader(conf)
# mock upload_file to fail, and clean up mock
def mock_upload_file(self, *args, **kwargs):
uploader.uploaded_files.pop()
return False
uploader.internal_proxy.upload_file = mock_upload_file
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
# file still there
self.assertEquals(len(os.listdir(t)), 1)
def test_error_parsing(self):
fname = 'container-sync-realms.conf'
fcontents = 'invalid'
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(
logger.all_log_lines(),
{'error': [
"Could not load '%s': File contains no section headers.\n"
"file: %s, line: 1\n"
"'invalid'" % (fpath, fpath)]})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), [])
def test_upload_file_failed(self):
files = ['plugin-%s' % datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA]) as t:
conf = {'log_dir': t, 'unlink_log': 'true',
'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
# mock upload_file to fail, and clean up mock
def mock_upload_file(self, *args, **kwargs):
uploader.uploaded_files.pop()
return False
uploader.internal_proxy.upload_file = mock_upload_file
self.assertRaises(SystemExit, uploader.run_once)
# file still there
self.assertEquals(len(os.listdir(t)), 1)
def test_init_request_processor(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
app, conf, logger, log_name = wsgi.init_request_processor(
conf_file, 'proxy-server')
# verify pipeline is catch_errors -> dlo -> proxy-server
expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assert_(isinstance(app, expected))
app = app.app
expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware
self.assert_(isinstance(app, expected))
app = app.app
expected = swift.common.middleware.dlo.DynamicLargeObject
self.assert_(isinstance(app, expected))
app = app.app
expected = swift.proxy.server.Application
self.assert_(isinstance(app, expected))
# config settings applied to app instance
self.assertEquals(0.2, app.conn_timeout)
# appconfig returns values from 'proxy-server' section
expected = {
'__file__': conf_file,
'here': os.path.dirname(conf_file),
'conn_timeout': '0.2',
'swift_dir': t,
}
self.assertEquals(expected, conf)
# logger works
logger.info('testing')
self.assertEquals('proxy-server', log_name)
def test_init_request_processor(self):
config = """
[DEFAULT]
swift_dir = TEMPDIR
[pipeline:main]
pipeline = proxy-server
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
"""
contents = dedent(config)
with temptree(['proxy-server.conf']) as t:
conf_file = os.path.join(t, 'proxy-server.conf')
with open(conf_file, 'w') as f:
f.write(contents.replace('TEMPDIR', t))
_fake_rings(t)
app, conf, logger, log_name = wsgi.init_request_processor(
conf_file, 'proxy-server')
# verify pipeline is catch_errors -> dlo -> proxy-server
expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assert_(isinstance(app, expected))
app = app.app
expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware
self.assert_(isinstance(app, expected))
app = app.app
expected = swift.common.middleware.dlo.DynamicLargeObject
self.assert_(isinstance(app, expected))
app = app.app
expected = swift.proxy.server.Application
self.assert_(isinstance(app, expected))
# config settings applied to app instance
self.assertEquals(0.2, app.conn_timeout)
# appconfig returns values from 'proxy-server' section
expected = {
'__file__': conf_file,
'here': os.path.dirname(conf_file),
'conn_timeout': '0.2',
'swift_dir': t,
}
self.assertEquals(expected, conf)
# logger works
logger.info('testing')
self.assertEquals('proxy-server', log_name)
def test_bad_mtime_check_interval(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[DEFAULT]
mtime_check_interval = invalid
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(
logger.lines_dict,
{'error': [
"Error in '%s' with mtime_check_interval: invalid literal "
"for int() with base 10: 'invalid'" % fpath]})
self.assertEqual(csr.mtime_check_interval, 300)
def test_write_file(self):
with temptree([]) as t:
file_name = os.path.join(t, 'test')
utils.write_file(file_name, 'test')
with open(file_name, 'r') as f:
contents = f.read()
self.assertEquals(contents, 'test')
# and also subdirs
file_name = os.path.join(t, 'subdir/test2')
utils.write_file(file_name, 'test2')
with open(file_name, 'r') as f:
contents = f.read()
self.assertEquals(contents, 'test2')
# but can't over-write files
file_name = os.path.join(t, 'subdir/test2/test3')
self.assertRaises(IOError, utils.write_file, file_name, 'test3')
def test_unlink_log(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA]) as t:
conf = {'log_dir': t, 'unlink_log': 'false'}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
# file still there
self.assertEquals(len(os.listdir(t)), 1)
conf = {'log_dir': t, 'unlink_log': 'true'}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
# file gone
self.assertEquals(len(os.listdir(t)), 0)
def test_upload_file_failed(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA]) as t:
conf = {'log_dir': t, 'unlink_log': 'true'}
uploader = MockLogUploader(conf)
# mock upload_file to fail, and clean up mock
def mock_upload_file(self, *args, **kwargs):
uploader.uploaded_files.pop()
return False
uploader.internal_proxy.upload_file = mock_upload_file
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
# file still there
self.assertEquals(len(os.listdir(t)), 1)
def test_empty_realm(self):
fname = "container-sync-realms.conf"
fcontents = """
[US]
"""
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.lines_dict, {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), ["US"])
self.assertEqual(csr.key("US"), None)
self.assertEqual(csr.key2("US"), None)
self.assertEqual(csr.clusters("US"), [])
self.assertEqual(csr.endpoint("US", "JUST_TESTING"), None)
def test_empty_realm(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[US]
'''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
self.assertEqual(logger.all_log_lines(), {})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), ['US'])
self.assertIsNone(csr.key('US'))
self.assertIsNone(csr.key2('US'))
self.assertEqual(csr.clusters('US'), [])
self.assertIsNone(csr.endpoint('US', 'JUST_TESTING'))
def test_init_request_processor_from_conf_dir(self):
config_dir = {
'proxy-server.conf.d/pipeline.conf':
"""
[pipeline:main]
pipeline = catch_errors proxy-server
""",
'proxy-server.conf.d/app.conf':
"""
[app:proxy-server]
use = egg:swift#proxy
conn_timeout = 0.2
""",
'proxy-server.conf.d/catch-errors.conf':
"""
[filter:catch_errors]
use = egg:swift#catch_errors
"""
}
# strip indent from test config contents
config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items())
with mock.patch('swift.proxy.server.Application.modify_wsgi_pipeline'):
with temptree(*zip(*config_dir.items())) as conf_root:
conf_dir = os.path.join(conf_root, 'proxy-server.conf.d')
with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f:
f.write('[DEFAULT]\nswift_dir = %s' % conf_root)
_fake_rings(conf_root)
app, conf, logger, log_name = wsgi.init_request_processor(
conf_dir, 'proxy-server')
# verify pipeline is catch_errors -> proxy-server
expected = swift.common.middleware.catch_errors.CatchErrorMiddleware
self.assert_(isinstance(app, expected))
self.assert_(isinstance(app.app, swift.proxy.server.Application))
# config settings applied to app instance
self.assertEquals(0.2, app.app.conn_timeout)
# appconfig returns values from 'proxy-server' section
expected = {
'__file__': conf_dir,
'here': conf_dir,
'conn_timeout': '0.2',
'swift_dir': conf_root,
}
self.assertEquals(expected, conf)
# logger works
logger.info('testing')
self.assertEquals('proxy-server', log_name)
def test_bad_mtime_check_interval(self):
fname = 'container-sync-realms.conf'
fcontents = '''
[DEFAULT]
mtime_check_interval = invalid
'''
with temptree([fname], [fcontents]) as tempdir:
logger = debug_logger()
fpath = os.path.join(tempdir, fname)
csr = ContainerSyncRealms(fpath, logger)
logs = logger.all_log_lines()
self.assertEqual(logs, {'error': [ANY]})
line = logs['error'][0]
self.assertIn(
"Error in '%s' with mtime_check_interval: "
"could not convert string to float:" % fpath, line)
self.assertEqual(csr.mtime_check_interval, 300)
def test_os_error(self):
fname = 'container-sync-realms.conf'
fcontents = ''
with temptree([fname], [fcontents]) as tempdir:
logger = FakeLogger()
fpath = os.path.join(tempdir, fname)
os.chmod(tempdir, 0)
csr = ContainerSyncRealms(fpath, logger)
try:
self.assertEqual(
logger.lines_dict,
{'error': [
"Could not load '%s': [Errno 13] Permission denied: "
"'%s'" % (fpath, fpath)]})
self.assertEqual(csr.mtime_check_interval, 300)
self.assertEqual(csr.realms(), [])
finally:
os.chmod(tempdir, 0700)
