def test_finding_defaults_from_file(self):
# If DLO has no config vars, go pull them from the proxy server's
# config section
proxy_conf = dedent("""
[DEFAULT]
bind_ip = 10.4.5.6
[pipeline:main]
pipeline = catch_errors dlo ye-olde-proxy-server
[filter:dlo]
use = egg:swift#dlo
[app:ye-olde-proxy-server]
use = egg:swift#proxy
rate_limit_after_segment = 13
max_get_time = 2900
""")
conffile = tempfile.NamedTemporaryFile()
conffile.write(proxy_conf)
conffile.flush()
mware = dlo.filter_factory({
'__file__': conffile.name
})("no app here")
self.assertEqual(1, mware.rate_limit_segments_per_sec)
self.assertEqual(13, mware.rate_limit_after_segment)
self.assertEqual(2900, mware.max_get_time)
def test_finding_defaults_from_file(self):
# If DLO has no config vars, go pull them from the proxy server's
# config section
proxy_conf = dedent("""
[DEFAULT]
bind_ip = 10.4.5.6
[pipeline:main]
pipeline = catch_errors dlo ye-olde-proxy-server
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:dlo]
use = egg:swift#dlo
[app:ye-olde-proxy-server]
use = egg:swift#proxy
rate_limit_after_segment = 13
set max_get_time = 2900
""")
conffile = tempfile.NamedTemporaryFile(mode='w')
conffile.write(proxy_conf)
conffile.flush()
mware = dlo.filter_factory({'__file__': conffile.name})("no app here")
self.assertEqual(1, mware.rate_limit_segments_per_sec)
self.assertEqual(13, mware.rate_limit_after_segment)
self.assertEqual(2900, mware.max_get_time)
conffile.close()
def test_skip_defaults_if_configured(self):
# The presence of even one config value in our config section means we
# won't go looking for the proxy config at all.
proxy_conf = dedent("""
[DEFAULT]
bind_ip = 10.4.5.6
[pipeline:main]
pipeline = catch_errors dlo ye-olde-proxy-server
[filter:dlo]
use = egg:swift#dlo
max_get_time = 3600
[app:ye-olde-proxy-server]
use = egg:swift#proxy
rate_limit_segments_per_sec = 7
rate_limit_after_segment = 13
max_get_time = 2900
""")
conffile = tempfile.NamedTemporaryFile()
conffile.write(proxy_conf)
conffile.flush()
mware = dlo.filter_factory({
'max_get_time': '3600',
'__file__': conffile.name
})("no app here")
self.assertEqual(1, mware.rate_limit_segments_per_sec)
self.assertEqual(10, mware.rate_limit_after_segment)
self.assertEqual(3600, mware.max_get_time)
def test_skip_defaults_if_configured(self):
# The presence of even one config value in our config section means we
# won't go looking for the proxy config at all.
proxy_conf = dedent("""
[DEFAULT]
bind_ip = 10.4.5.6
[pipeline:main]
pipeline = catch_errors dlo ye-olde-proxy-server
[filter:dlo]
use = egg:swift#dlo
max_get_time = 3600
[app:ye-olde-proxy-server]
use = egg:swift#proxy
rate_limit_segments_per_sec = 7
rate_limit_after_segment = 13
max_get_time = 2900
""")
conffile = tempfile.NamedTemporaryFile()
conffile.write(proxy_conf)
conffile.flush()
mware = dlo.filter_factory({
'max_get_time': '3600',
'__file__': conffile.name
})("no app here")
self.assertEqual(1, mware.rate_limit_segments_per_sec)
self.assertEqual(10, mware.rate_limit_after_segment)
self.assertEqual(3600, mware.max_get_time)
def setUp(self):
super(TestDloCopyHook, self).setUp()
self.app.register('GET', '/v1/AUTH_test/c/o1', swob.HTTPOk, {
'Content-Length': '10',
'Etag': 'o1-etag'
}, "aaaaaaaaaa")
self.app.register('GET', '/v1/AUTH_test/c/o2', swob.HTTPOk, {
'Content-Length': '10',
'Etag': 'o2-etag'
}, "bbbbbbbbbb")
self.app.register('GET', '/v1/AUTH_test/c/man', swob.HTTPOk,
{'X-Object-Manifest': 'c/o'}, "manifest-contents")
lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream'
segs = [{
"hash": "o1-etag",
"bytes": 10,
"name": "o1",
"last_modified": lm,
"content_type": ct
}, {
"hash": "o2-etag",
"bytes": 5,
"name": "o2",
"last_modified": lm,
"content_type": ct
}]
self.app.register('GET', '/v1/AUTH_test/c?format=json&prefix=o',
swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs))
copy_hook = [None]
# slip this guy in there to pull out the hook
def extract_copy_hook(env, sr):
copy_hook[0] = env.get('swift.copy_hook')
return self.app(env, sr)
self.dlo = dlo.filter_factory({})(extract_copy_hook)
req = swob.Request.blank('/v1/AUTH_test/c/o1',
environ={'REQUEST_METHOD': 'GET'})
self.dlo(req.environ, fake_start_response)
self.copy_hook = copy_hook[0]
self.assertTrue(self.copy_hook is not None) # sanity check
def test_finding_defaults_from_dir(self):
# If DLO has no config vars, go pull them from the proxy server's
# config section
proxy_conf1 = dedent("""
[DEFAULT]
bind_ip = 10.4.5.6
[pipeline:main]
pipeline = catch_errors dlo ye-olde-proxy-server
""")
proxy_conf2 = dedent("""
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:dlo]
use = egg:swift#dlo
[app:ye-olde-proxy-server]
use = egg:swift#proxy
rate_limit_after_segment = 13
max_get_time = 2900
""")
conf_dir = self.tmpdir
conffile1 = tempfile.NamedTemporaryFile(mode='w',
dir=conf_dir, suffix='.conf')
conffile1.write(proxy_conf1)
conffile1.flush()
conffile2 = tempfile.NamedTemporaryFile(mode='w',
dir=conf_dir, suffix='.conf')
conffile2.write(proxy_conf2)
conffile2.flush()
mware = dlo.filter_factory({
'__file__': conf_dir
})("no app here")
self.assertEqual(1, mware.rate_limit_segments_per_sec)
self.assertEqual(13, mware.rate_limit_after_segment)
self.assertEqual(2900, mware.max_get_time)
conffile1.close()
conffile2.close()
def setUp(self):
super(TestDloCopyHook, self).setUp()
self.app.register(
'GET', '/v1/AUTH_test/c/o1', swob.HTTPOk,
{'Content-Length': '10', 'Etag': 'o1-etag'},
"aaaaaaaaaa")
self.app.register(
'GET', '/v1/AUTH_test/c/o2', swob.HTTPOk,
{'Content-Length': '10', 'Etag': 'o2-etag'},
"bbbbbbbbbb")
self.app.register(
'GET', '/v1/AUTH_test/c/man',
swob.HTTPOk, {'X-Object-Manifest': 'c/o'},
"manifest-contents")
lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream'
segs = [{"hash": "o1-etag", "bytes": 10, "name": "o1",
"last_modified": lm, "content_type": ct},
{"hash": "o2-etag", "bytes": 5, "name": "o2",
"last_modified": lm, "content_type": ct}]
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=o',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs))
copy_hook = [None]
# slip this guy in there to pull out the hook
def extract_copy_hook(env, sr):
copy_hook[0] = env.get('swift.copy_response_hook')
return self.app(env, sr)
self.dlo = dlo.filter_factory({})(extract_copy_hook)
req = swob.Request.blank('/v1/AUTH_test/c/o1',
environ={'REQUEST_METHOD': 'GET'})
self.dlo(req.environ, fake_start_response)
self.copy_hook = copy_hook[0]
self.assertTrue(self.copy_hook is not None) # sanity check
def test_finding_defaults_from_dir(self):
# If DLO has no config vars, go pull them from the proxy server's
# config section
proxy_conf1 = dedent("""
[DEFAULT]
bind_ip = 10.4.5.6
[pipeline:main]
pipeline = catch_errors dlo ye-olde-proxy-server
""")
proxy_conf2 = dedent("""
[filter:catch_errors]
use = egg:swift#catch_errors
[filter:dlo]
use = egg:swift#dlo
[app:ye-olde-proxy-server]
use = egg:swift#proxy
rate_limit_after_segment = 13
max_get_time = 2900
""")
conf_dir = self.tmpdir
conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
conffile1.write(proxy_conf1)
conffile1.flush()
conffile2 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf')
conffile2.write(proxy_conf2)
conffile2.flush()
mware = dlo.filter_factory({
'__file__': conf_dir
})("no app here")
self.assertEqual(1, mware.rate_limit_segments_per_sec)
self.assertEqual(13, mware.rate_limit_after_segment)
self.assertEqual(2900, mware.max_get_time)
def setUp(self):
super(TestDloCopyHook, self).setUp()
self.app.register(
"GET", "/v1/AUTH_test/c/o1", swob.HTTPOk, {"Content-Length": "10", "Etag": "o1-etag"}, "aaaaaaaaaa"
)
self.app.register(
"GET", "/v1/AUTH_test/c/o2", swob.HTTPOk, {"Content-Length": "10", "Etag": "o2-etag"}, "bbbbbbbbbb"
)
self.app.register("GET", "/v1/AUTH_test/c/man", swob.HTTPOk, {"X-Object-Manifest": "c/o"}, "manifest-contents")
lm = "2013-11-22T02:42:13.781760"
ct = "application/octet-stream"
segs = [
{"hash": "o1-etag", "bytes": 10, "name": "o1", "last_modified": lm, "content_type": ct},
{"hash": "o2-etag", "bytes": 5, "name": "o2", "last_modified": lm, "content_type": ct},
]
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=o",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(segs),
)
copy_hook = [None]
# slip this guy in there to pull out the hook
def extract_copy_hook(env, sr):
copy_hook[0] = env.get("swift.copy_response_hook")
return self.app(env, sr)
self.dlo = dlo.filter_factory({})(extract_copy_hook)
req = swob.Request.blank("/v1/AUTH_test/c/o1", environ={"REQUEST_METHOD": "GET"})
self.dlo(req.environ, fake_start_response)
self.copy_hook = copy_hook[0]
self.assertTrue(self.copy_hook is not None) # sanity check
def setUp(self):
self.app = FakeSwift()
self.dlo = dlo.filter_factory({
# don't slow down tests with rate limiting
'rate_limit_after_segment': '1000000',
})(self.app)
self.app.register(
'GET', '/v1/AUTH_test/c/seg_01',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg01-etag'},
'aaaaa')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_02',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg02-etag'},
'bbbbb')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_03',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg03-etag'},
'ccccc')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_04',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg04-etag'},
'ddddd')
self.app.register(
'GET', '/v1/AUTH_test/c/seg_05',
swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg05-etag'},
'eeeee')
# an unrelated object (not seg*) to test the prefix matching
self.app.register(
'GET', '/v1/AUTH_test/c/catpicture.jpg',
swob.HTTPOk, {'Content-Length': '9', 'Etag': 'cats-etag'},
'meow meow meow meow')
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest',
swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag',
'X-Object-Manifest': 'c/seg'},
'manifest-contents')
lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream'
segs = [{"hash": "seg01-etag", "bytes": 5, "name": "seg_01",
"last_modified": lm, "content_type": ct},
{"hash": "seg02-etag", "bytes": 5, "name": "seg_02",
"last_modified": lm, "content_type": ct},
{"hash": "seg03-etag", "bytes": 5, "name": "seg_03",
"last_modified": lm, "content_type": ct},
{"hash": "seg04-etag", "bytes": 5, "name": "seg_04",
"last_modified": lm, "content_type": ct},
{"hash": "seg05-etag", "bytes": 5, "name": "seg_05",
"last_modified": lm, "content_type": ct}]
full_container_listing = segs + [{"hash": "cats-etag", "bytes": 9,
"name": "catpicture.jpg",
"last_modified": lm,
"content_type": "application/png"}]
self.app.register(
'GET', '/v1/AUTH_test/c?format=json',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(full_container_listing))
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=seg',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs))
# This is to let us test multi-page container listings; we use the
# trailing underscore to send small (pagesize=3) listings.
#
# If you're testing against this, be sure to mock out
# CONTAINER_LISTING_LIMIT to 3 in your test.
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-many-segments',
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg',
'X-Object-Manifest': 'c/seg_'},
'manyseg')
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=seg_',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[:3]))
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[3:]))
# Here's a manifest with 0 segments
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-no-segments',
swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg',
'X-Object-Manifest': 'c/noseg_'},
'noseg')
self.app.register(
'GET', '/v1/AUTH_test/c?format=json&prefix=noseg_',
swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
json.dumps([]))
def setUp(self):
self.app = FakeSwift()
self.dlo = dlo.filter_factory({
# don't slow down tests with rate limiting
'rate_limit_after_segment': '1000000',
})(self.app)
self.dlo.logger = self.app.logger
self.app.register('GET', '/v1/AUTH_test/c/seg_01', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("aaaaa")
}, b'aaaaa')
self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("bbbbb")
}, b'bbbbb')
self.app.register('GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("ccccc")
}, b'ccccc')
self.app.register('GET', '/v1/AUTH_test/c/seg_04', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("ddddd")
}, b'ddddd')
self.app.register('GET', '/v1/AUTH_test/c/seg_05', swob.HTTPOk, {
'Content-Length': '5',
'Etag': md5hex("eeeee")
}, b'eeeee')
# an unrelated object (not seg*) to test the prefix matching
self.app.register('GET', '/v1/AUTH_test/c/catpicture.jpg', swob.HTTPOk,
{
'Content-Length': '9',
'Etag': md5hex("meow meow meow meow")
}, b'meow meow meow meow')
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest', swob.HTTPOk, {
'Content-Length': '17',
'Etag': 'manifest-etag',
'X-Object-Manifest': 'c/seg'
}, b'manifest-contents')
lm = '2013-11-22T02:42:13.781760'
ct = 'application/octet-stream'
segs = [{
"hash": md5hex("aaaaa"),
"bytes": 5,
"name": "seg_01",
"last_modified": lm,
"content_type": ct
}, {
"hash": md5hex("bbbbb"),
"bytes": 5,
"name": "seg_02",
"last_modified": lm,
"content_type": ct
}, {
"hash": md5hex("ccccc"),
"bytes": 5,
"name": "seg_03",
"last_modified": lm,
"content_type": ct
}, {
"hash": md5hex("ddddd"),
"bytes": 5,
"name": "seg_04",
"last_modified": lm,
"content_type": ct
}, {
"hash": md5hex("eeeee"),
"bytes": 5,
"name": "seg_05",
"last_modified": lm,
"content_type": ct
}]
full_container_listing = segs + [{
"hash": "cats-etag",
"bytes": 9,
"name": "catpicture.jpg",
"last_modified": lm,
"content_type": "application/png"
}]
self.app.register('GET', '/v1/AUTH_test/c', swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(full_container_listing).encode('ascii'))
self.app.register('GET', '/v1/AUTH_test/c?prefix=seg', swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs).encode('ascii'))
# This is to let us test multi-page container listings; we use the
# trailing underscore to send small (pagesize=3) listings.
#
# If you're testing against this, be sure to mock out
# CONTAINER_LISTING_LIMIT to 3 in your test.
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-many-segments', swob.HTTPOk,
{
'Content-Length': '7',
'Etag': 'etag-manyseg',
'X-Object-Manifest': 'c/seg_'
}, b'manyseg')
self.app.register('GET', '/v1/AUTH_test/c?prefix=seg_', swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[:3]).encode('ascii'))
self.app.register('GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03',
swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps(segs[3:]).encode('ascii'))
# Here's a manifest with 0 segments
self.app.register(
'GET', '/v1/AUTH_test/mancon/manifest-no-segments', swob.HTTPOk, {
'Content-Length': '7',
'Etag': 'noseg',
'X-Object-Manifest': 'c/noseg_'
}, b'noseg')
self.app.register('GET', '/v1/AUTH_test/c?prefix=noseg_', swob.HTTPOk,
{'Content-Type': 'application/json; charset=utf-8'},
json.dumps([]).encode('ascii'))
def setUp(self):
self.app = FakeSwift()
self.dlo = dlo.filter_factory(
{
# don't slow down tests with rate limiting
"rate_limit_after_segment": "1000000"
}
)(self.app)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_01", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg01-etag"}, "aaaaa"
)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_02", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg02-etag"}, "bbbbb"
)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_03", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg03-etag"}, "ccccc"
)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_04", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg04-etag"}, "ddddd"
)
self.app.register(
"GET", "/v1/AUTH_test/c/seg_05", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg05-etag"}, "eeeee"
)
# an unrelated object (not seg*) to test the prefix matching
self.app.register(
"GET",
"/v1/AUTH_test/c/catpicture.jpg",
swob.HTTPOk,
{"Content-Length": "9", "Etag": "cats-etag"},
"meow meow meow meow",
)
self.app.register(
"GET",
"/v1/AUTH_test/mancon/manifest",
swob.HTTPOk,
{"Content-Length": "17", "Etag": "manifest-etag", "X-Object-Manifest": "c/seg"},
"manifest-contents",
)
lm = "2013-11-22T02:42:13.781760"
ct = "application/octet-stream"
segs = [
{"hash": "seg01-etag", "bytes": 5, "name": "seg_01", "last_modified": lm, "content_type": ct},
{"hash": "seg02-etag", "bytes": 5, "name": "seg_02", "last_modified": lm, "content_type": ct},
{"hash": "seg03-etag", "bytes": 5, "name": "seg_03", "last_modified": lm, "content_type": ct},
{"hash": "seg04-etag", "bytes": 5, "name": "seg_04", "last_modified": lm, "content_type": ct},
{"hash": "seg05-etag", "bytes": 5, "name": "seg_05", "last_modified": lm, "content_type": ct},
]
full_container_listing = segs + [
{
"hash": "cats-etag",
"bytes": 9,
"name": "catpicture.jpg",
"last_modified": lm,
"content_type": "application/png",
}
]
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(full_container_listing),
)
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=seg",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(segs),
)
# This is to let us test multi-page container listings; we use the
# trailing underscore to send small (pagesize=3) listings.
#
# If you're testing against this, be sure to mock out
# CONTAINER_LISTING_LIMIT to 3 in your test.
self.app.register(
"GET",
"/v1/AUTH_test/mancon/manifest-many-segments",
swob.HTTPOk,
{"Content-Length": "7", "Etag": "etag-manyseg", "X-Object-Manifest": "c/seg_"},
"manyseg",
)
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=seg_",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(segs[:3]),
)
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps(segs[3:]),
)
# Here's a manifest with 0 segments
self.app.register(
"GET",
"/v1/AUTH_test/mancon/manifest-no-segments",
swob.HTTPOk,
{"Content-Length": "7", "Etag": "noseg", "X-Object-Manifest": "c/noseg_"},
"noseg",
)
self.app.register(
"GET",
"/v1/AUTH_test/c?format=json&prefix=noseg_",
swob.HTTPOk,
{"Content-Type": "application/json; charset=utf-8"},
json.dumps([]),
)
