def test_finding_defaults_from_file(self): # If DLO has no config vars, go pull them from the proxy server's # config section proxy_conf = dedent(""" [DEFAULT] bind_ip = 10.4.5.6 [pipeline:main] pipeline = catch_errors dlo ye-olde-proxy-server [filter:dlo] use = egg:swift#dlo [app:ye-olde-proxy-server] use = egg:swift#proxy rate_limit_after_segment = 13 max_get_time = 2900 """) conffile = tempfile.NamedTemporaryFile() conffile.write(proxy_conf) conffile.flush() mware = dlo.filter_factory({ '__file__': conffile.name })("no app here") self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(2900, mware.max_get_time)
def test_finding_defaults_from_file(self): # If DLO has no config vars, go pull them from the proxy server's # config section proxy_conf = dedent(""" [DEFAULT] bind_ip = 10.4.5.6 [pipeline:main] pipeline = catch_errors dlo ye-olde-proxy-server [filter:catch_errors] use = egg:swift#catch_errors [filter:dlo] use = egg:swift#dlo [app:ye-olde-proxy-server] use = egg:swift#proxy rate_limit_after_segment = 13 set max_get_time = 2900 """) conffile = tempfile.NamedTemporaryFile(mode='w') conffile.write(proxy_conf) conffile.flush() mware = dlo.filter_factory({'__file__': conffile.name})("no app here") self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(2900, mware.max_get_time) conffile.close()
def test_skip_defaults_if_configured(self): # The presence of even one config value in our config section means we # won't go looking for the proxy config at all. proxy_conf = dedent(""" [DEFAULT] bind_ip = 10.4.5.6 [pipeline:main] pipeline = catch_errors dlo ye-olde-proxy-server [filter:dlo] use = egg:swift#dlo max_get_time = 3600 [app:ye-olde-proxy-server] use = egg:swift#proxy rate_limit_segments_per_sec = 7 rate_limit_after_segment = 13 max_get_time = 2900 """) conffile = tempfile.NamedTemporaryFile() conffile.write(proxy_conf) conffile.flush() mware = dlo.filter_factory({ 'max_get_time': '3600', '__file__': conffile.name })("no app here") self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(10, mware.rate_limit_after_segment) self.assertEqual(3600, mware.max_get_time)
def setUp(self): super(TestDloCopyHook, self).setUp() self.app.register('GET', '/v1/AUTH_test/c/o1', swob.HTTPOk, { 'Content-Length': '10', 'Etag': 'o1-etag' }, "aaaaaaaaaa") self.app.register('GET', '/v1/AUTH_test/c/o2', swob.HTTPOk, { 'Content-Length': '10', 'Etag': 'o2-etag' }, "bbbbbbbbbb") self.app.register('GET', '/v1/AUTH_test/c/man', swob.HTTPOk, {'X-Object-Manifest': 'c/o'}, "manifest-contents") lm = '2013-11-22T02:42:13.781760' ct = 'application/octet-stream' segs = [{ "hash": "o1-etag", "bytes": 10, "name": "o1", "last_modified": lm, "content_type": ct }, { "hash": "o2-etag", "bytes": 5, "name": "o2", "last_modified": lm, "content_type": ct }] self.app.register('GET', '/v1/AUTH_test/c?format=json&prefix=o', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs)) copy_hook = [None] # slip this guy in there to pull out the hook def extract_copy_hook(env, sr): copy_hook[0] = env.get('swift.copy_hook') return self.app(env, sr) self.dlo = dlo.filter_factory({})(extract_copy_hook) req = swob.Request.blank('/v1/AUTH_test/c/o1', environ={'REQUEST_METHOD': 'GET'}) self.dlo(req.environ, fake_start_response) self.copy_hook = copy_hook[0] self.assertTrue(self.copy_hook is not None) # sanity check
def test_finding_defaults_from_dir(self): # If DLO has no config vars, go pull them from the proxy server's # config section proxy_conf1 = dedent(""" [DEFAULT] bind_ip = 10.4.5.6 [pipeline:main] pipeline = catch_errors dlo ye-olde-proxy-server """) proxy_conf2 = dedent(""" [filter:catch_errors] use = egg:swift#catch_errors [filter:dlo] use = egg:swift#dlo [app:ye-olde-proxy-server] use = egg:swift#proxy rate_limit_after_segment = 13 max_get_time = 2900 """) conf_dir = self.tmpdir conffile1 = tempfile.NamedTemporaryFile(mode='w', dir=conf_dir, suffix='.conf') conffile1.write(proxy_conf1) conffile1.flush() conffile2 = tempfile.NamedTemporaryFile(mode='w', dir=conf_dir, suffix='.conf') conffile2.write(proxy_conf2) conffile2.flush() mware = dlo.filter_factory({ '__file__': conf_dir })("no app here") self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(2900, mware.max_get_time) conffile1.close() conffile2.close()
def setUp(self): super(TestDloCopyHook, self).setUp() self.app.register( 'GET', '/v1/AUTH_test/c/o1', swob.HTTPOk, {'Content-Length': '10', 'Etag': 'o1-etag'}, "aaaaaaaaaa") self.app.register( 'GET', '/v1/AUTH_test/c/o2', swob.HTTPOk, {'Content-Length': '10', 'Etag': 'o2-etag'}, "bbbbbbbbbb") self.app.register( 'GET', '/v1/AUTH_test/c/man', swob.HTTPOk, {'X-Object-Manifest': 'c/o'}, "manifest-contents") lm = '2013-11-22T02:42:13.781760' ct = 'application/octet-stream' segs = [{"hash": "o1-etag", "bytes": 10, "name": "o1", "last_modified": lm, "content_type": ct}, {"hash": "o2-etag", "bytes": 5, "name": "o2", "last_modified": lm, "content_type": ct}] self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=o', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs)) copy_hook = [None] # slip this guy in there to pull out the hook def extract_copy_hook(env, sr): copy_hook[0] = env.get('swift.copy_response_hook') return self.app(env, sr) self.dlo = dlo.filter_factory({})(extract_copy_hook) req = swob.Request.blank('/v1/AUTH_test/c/o1', environ={'REQUEST_METHOD': 'GET'}) self.dlo(req.environ, fake_start_response) self.copy_hook = copy_hook[0] self.assertTrue(self.copy_hook is not None) # sanity check
def test_finding_defaults_from_dir(self): # If DLO has no config vars, go pull them from the proxy server's # config section proxy_conf1 = dedent(""" [DEFAULT] bind_ip = 10.4.5.6 [pipeline:main] pipeline = catch_errors dlo ye-olde-proxy-server """) proxy_conf2 = dedent(""" [filter:catch_errors] use = egg:swift#catch_errors [filter:dlo] use = egg:swift#dlo [app:ye-olde-proxy-server] use = egg:swift#proxy rate_limit_after_segment = 13 max_get_time = 2900 """) conf_dir = self.tmpdir conffile1 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf') conffile1.write(proxy_conf1) conffile1.flush() conffile2 = tempfile.NamedTemporaryFile(dir=conf_dir, suffix='.conf') conffile2.write(proxy_conf2) conffile2.flush() mware = dlo.filter_factory({ '__file__': conf_dir })("no app here") self.assertEqual(1, mware.rate_limit_segments_per_sec) self.assertEqual(13, mware.rate_limit_after_segment) self.assertEqual(2900, mware.max_get_time)
def setUp(self): super(TestDloCopyHook, self).setUp() self.app.register( "GET", "/v1/AUTH_test/c/o1", swob.HTTPOk, {"Content-Length": "10", "Etag": "o1-etag"}, "aaaaaaaaaa" ) self.app.register( "GET", "/v1/AUTH_test/c/o2", swob.HTTPOk, {"Content-Length": "10", "Etag": "o2-etag"}, "bbbbbbbbbb" ) self.app.register("GET", "/v1/AUTH_test/c/man", swob.HTTPOk, {"X-Object-Manifest": "c/o"}, "manifest-contents") lm = "2013-11-22T02:42:13.781760" ct = "application/octet-stream" segs = [ {"hash": "o1-etag", "bytes": 10, "name": "o1", "last_modified": lm, "content_type": ct}, {"hash": "o2-etag", "bytes": 5, "name": "o2", "last_modified": lm, "content_type": ct}, ] self.app.register( "GET", "/v1/AUTH_test/c?format=json&prefix=o", swob.HTTPOk, {"Content-Type": "application/json; charset=utf-8"}, json.dumps(segs), ) copy_hook = [None] # slip this guy in there to pull out the hook def extract_copy_hook(env, sr): copy_hook[0] = env.get("swift.copy_response_hook") return self.app(env, sr) self.dlo = dlo.filter_factory({})(extract_copy_hook) req = swob.Request.blank("/v1/AUTH_test/c/o1", environ={"REQUEST_METHOD": "GET"}) self.dlo(req.environ, fake_start_response) self.copy_hook = copy_hook[0] self.assertTrue(self.copy_hook is not None) # sanity check
def setUp(self): self.app = FakeSwift() self.dlo = dlo.filter_factory({ # don't slow down tests with rate limiting 'rate_limit_after_segment': '1000000', })(self.app) self.app.register( 'GET', '/v1/AUTH_test/c/seg_01', swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg01-etag'}, 'aaaaa') self.app.register( 'GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg02-etag'}, 'bbbbb') self.app.register( 'GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg03-etag'}, 'ccccc') self.app.register( 'GET', '/v1/AUTH_test/c/seg_04', swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg04-etag'}, 'ddddd') self.app.register( 'GET', '/v1/AUTH_test/c/seg_05', swob.HTTPOk, {'Content-Length': '5', 'Etag': 'seg05-etag'}, 'eeeee') # an unrelated object (not seg*) to test the prefix matching self.app.register( 'GET', '/v1/AUTH_test/c/catpicture.jpg', swob.HTTPOk, {'Content-Length': '9', 'Etag': 'cats-etag'}, 'meow meow meow meow') self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest', swob.HTTPOk, {'Content-Length': '17', 'Etag': 'manifest-etag', 'X-Object-Manifest': 'c/seg'}, 'manifest-contents') lm = '2013-11-22T02:42:13.781760' ct = 'application/octet-stream' segs = [{"hash": "seg01-etag", "bytes": 5, "name": "seg_01", "last_modified": lm, "content_type": ct}, {"hash": "seg02-etag", "bytes": 5, "name": "seg_02", "last_modified": lm, "content_type": ct}, {"hash": "seg03-etag", "bytes": 5, "name": "seg_03", "last_modified": lm, "content_type": ct}, {"hash": "seg04-etag", "bytes": 5, "name": "seg_04", "last_modified": lm, "content_type": ct}, {"hash": "seg05-etag", "bytes": 5, "name": "seg_05", "last_modified": lm, "content_type": ct}] full_container_listing = segs + [{"hash": "cats-etag", "bytes": 9, "name": "catpicture.jpg", "last_modified": lm, "content_type": "application/png"}] self.app.register( 'GET', '/v1/AUTH_test/c?format=json', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(full_container_listing)) self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=seg', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs)) # This is to let us test multi-page container listings; we use the # trailing underscore to send small (pagesize=3) listings. # # If you're testing against this, be sure to mock out # CONTAINER_LISTING_LIMIT to 3 in your test. self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest-many-segments', swob.HTTPOk, {'Content-Length': '7', 'Etag': 'etag-manyseg', 'X-Object-Manifest': 'c/seg_'}, 'manyseg') self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=seg_', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs[:3])) self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs[3:])) # Here's a manifest with 0 segments self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest-no-segments', swob.HTTPOk, {'Content-Length': '7', 'Etag': 'noseg', 'X-Object-Manifest': 'c/noseg_'}, 'noseg') self.app.register( 'GET', '/v1/AUTH_test/c?format=json&prefix=noseg_', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps([]))
def setUp(self): self.app = FakeSwift() self.dlo = dlo.filter_factory({ # don't slow down tests with rate limiting 'rate_limit_after_segment': '1000000', })(self.app) self.dlo.logger = self.app.logger self.app.register('GET', '/v1/AUTH_test/c/seg_01', swob.HTTPOk, { 'Content-Length': '5', 'Etag': md5hex("aaaaa") }, b'aaaaa') self.app.register('GET', '/v1/AUTH_test/c/seg_02', swob.HTTPOk, { 'Content-Length': '5', 'Etag': md5hex("bbbbb") }, b'bbbbb') self.app.register('GET', '/v1/AUTH_test/c/seg_03', swob.HTTPOk, { 'Content-Length': '5', 'Etag': md5hex("ccccc") }, b'ccccc') self.app.register('GET', '/v1/AUTH_test/c/seg_04', swob.HTTPOk, { 'Content-Length': '5', 'Etag': md5hex("ddddd") }, b'ddddd') self.app.register('GET', '/v1/AUTH_test/c/seg_05', swob.HTTPOk, { 'Content-Length': '5', 'Etag': md5hex("eeeee") }, b'eeeee') # an unrelated object (not seg*) to test the prefix matching self.app.register('GET', '/v1/AUTH_test/c/catpicture.jpg', swob.HTTPOk, { 'Content-Length': '9', 'Etag': md5hex("meow meow meow meow") }, b'meow meow meow meow') self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest', swob.HTTPOk, { 'Content-Length': '17', 'Etag': 'manifest-etag', 'X-Object-Manifest': 'c/seg' }, b'manifest-contents') lm = '2013-11-22T02:42:13.781760' ct = 'application/octet-stream' segs = [{ "hash": md5hex("aaaaa"), "bytes": 5, "name": "seg_01", "last_modified": lm, "content_type": ct }, { "hash": md5hex("bbbbb"), "bytes": 5, "name": "seg_02", "last_modified": lm, "content_type": ct }, { "hash": md5hex("ccccc"), "bytes": 5, "name": "seg_03", "last_modified": lm, "content_type": ct }, { "hash": md5hex("ddddd"), "bytes": 5, "name": "seg_04", "last_modified": lm, "content_type": ct }, { "hash": md5hex("eeeee"), "bytes": 5, "name": "seg_05", "last_modified": lm, "content_type": ct }] full_container_listing = segs + [{ "hash": "cats-etag", "bytes": 9, "name": "catpicture.jpg", "last_modified": lm, "content_type": "application/png" }] self.app.register('GET', '/v1/AUTH_test/c', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(full_container_listing).encode('ascii')) self.app.register('GET', '/v1/AUTH_test/c?prefix=seg', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs).encode('ascii')) # This is to let us test multi-page container listings; we use the # trailing underscore to send small (pagesize=3) listings. # # If you're testing against this, be sure to mock out # CONTAINER_LISTING_LIMIT to 3 in your test. self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest-many-segments', swob.HTTPOk, { 'Content-Length': '7', 'Etag': 'etag-manyseg', 'X-Object-Manifest': 'c/seg_' }, b'manyseg') self.app.register('GET', '/v1/AUTH_test/c?prefix=seg_', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs[:3]).encode('ascii')) self.app.register('GET', '/v1/AUTH_test/c?prefix=seg_&marker=seg_03', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(segs[3:]).encode('ascii')) # Here's a manifest with 0 segments self.app.register( 'GET', '/v1/AUTH_test/mancon/manifest-no-segments', swob.HTTPOk, { 'Content-Length': '7', 'Etag': 'noseg', 'X-Object-Manifest': 'c/noseg_' }, b'noseg') self.app.register('GET', '/v1/AUTH_test/c?prefix=noseg_', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps([]).encode('ascii'))
def setUp(self): self.app = FakeSwift() self.dlo = dlo.filter_factory( { # don't slow down tests with rate limiting "rate_limit_after_segment": "1000000" } )(self.app) self.app.register( "GET", "/v1/AUTH_test/c/seg_01", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg01-etag"}, "aaaaa" ) self.app.register( "GET", "/v1/AUTH_test/c/seg_02", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg02-etag"}, "bbbbb" ) self.app.register( "GET", "/v1/AUTH_test/c/seg_03", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg03-etag"}, "ccccc" ) self.app.register( "GET", "/v1/AUTH_test/c/seg_04", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg04-etag"}, "ddddd" ) self.app.register( "GET", "/v1/AUTH_test/c/seg_05", swob.HTTPOk, {"Content-Length": "5", "Etag": "seg05-etag"}, "eeeee" ) # an unrelated object (not seg*) to test the prefix matching self.app.register( "GET", "/v1/AUTH_test/c/catpicture.jpg", swob.HTTPOk, {"Content-Length": "9", "Etag": "cats-etag"}, "meow meow meow meow", ) self.app.register( "GET", "/v1/AUTH_test/mancon/manifest", swob.HTTPOk, {"Content-Length": "17", "Etag": "manifest-etag", "X-Object-Manifest": "c/seg"}, "manifest-contents", ) lm = "2013-11-22T02:42:13.781760" ct = "application/octet-stream" segs = [ {"hash": "seg01-etag", "bytes": 5, "name": "seg_01", "last_modified": lm, "content_type": ct}, {"hash": "seg02-etag", "bytes": 5, "name": "seg_02", "last_modified": lm, "content_type": ct}, {"hash": "seg03-etag", "bytes": 5, "name": "seg_03", "last_modified": lm, "content_type": ct}, {"hash": "seg04-etag", "bytes": 5, "name": "seg_04", "last_modified": lm, "content_type": ct}, {"hash": "seg05-etag", "bytes": 5, "name": "seg_05", "last_modified": lm, "content_type": ct}, ] full_container_listing = segs + [ { "hash": "cats-etag", "bytes": 9, "name": "catpicture.jpg", "last_modified": lm, "content_type": "application/png", } ] self.app.register( "GET", "/v1/AUTH_test/c?format=json", swob.HTTPOk, {"Content-Type": "application/json; charset=utf-8"}, json.dumps(full_container_listing), ) self.app.register( "GET", "/v1/AUTH_test/c?format=json&prefix=seg", swob.HTTPOk, {"Content-Type": "application/json; charset=utf-8"}, json.dumps(segs), ) # This is to let us test multi-page container listings; we use the # trailing underscore to send small (pagesize=3) listings. # # If you're testing against this, be sure to mock out # CONTAINER_LISTING_LIMIT to 3 in your test. self.app.register( "GET", "/v1/AUTH_test/mancon/manifest-many-segments", swob.HTTPOk, {"Content-Length": "7", "Etag": "etag-manyseg", "X-Object-Manifest": "c/seg_"}, "manyseg", ) self.app.register( "GET", "/v1/AUTH_test/c?format=json&prefix=seg_", swob.HTTPOk, {"Content-Type": "application/json; charset=utf-8"}, json.dumps(segs[:3]), ) self.app.register( "GET", "/v1/AUTH_test/c?format=json&prefix=seg_&marker=seg_03", swob.HTTPOk, {"Content-Type": "application/json; charset=utf-8"}, json.dumps(segs[3:]), ) # Here's a manifest with 0 segments self.app.register( "GET", "/v1/AUTH_test/mancon/manifest-no-segments", swob.HTTPOk, {"Content-Length": "7", "Etag": "noseg", "X-Object-Manifest": "c/noseg_"}, "noseg", ) self.app.register( "GET", "/v1/AUTH_test/c?format=json&prefix=noseg_", swob.HTTPOk, {"Content-Type": "application/json; charset=utf-8"}, json.dumps([]), )