def test_non_dir_contents(self): with temptree([]) as tmpdir: data = os.path.join(tmpdir, "drive", "data") os.makedirs(data) with open(os.path.join(data, "partition1"), "w"): pass partition = os.path.join(data, "partition2") os.makedirs(partition) with open(os.path.join(partition, "suffix1"), "w"): pass suffix = os.path.join(partition, "suffix2") os.makedirs(suffix) with open(os.path.join(suffix, "hash1"), "w"): pass locations = ondisk.audit_location_generator( tmpdir, "data", mount_check=False ) self.assertEqual(list(locations), [])
def test_search_tree(self): # file match & ext miss with temptree(["asdf.conf", "blarg.conf", "asdf.cfg"]) as t: asdf = utils.search_tree(t, "a*", ".conf") self.assertEquals(len(asdf), 1) self.assertEquals(asdf[0], os.path.join(t, "asdf.conf")) # multi-file match & glob miss & sort with temptree(["application.bin", "apple.bin", "apropos.bin"]) as t: app_bins = utils.search_tree(t, "app*", "bin") self.assertEquals(len(app_bins), 2) self.assertEquals(app_bins[0], os.path.join(t, "apple.bin")) self.assertEquals(app_bins[1], os.path.join(t, "application.bin")) # test file in folder & ext miss & glob miss files = ("sub/file1.ini", "sub/file2.conf", "sub.bin", "bus.ini", "bus/file3.ini") with temptree(files) as t: sub_ini = utils.search_tree(t, "sub*", ".ini") self.assertEquals(len(sub_ini), 1) self.assertEquals(sub_ini[0], os.path.join(t, "sub/file1.ini")) # test multi-file in folder & sub-folder & ext miss & glob miss files = ("folder_file.txt", "folder/1.txt", "folder/sub/2.txt", "folder2/3.txt", "Folder3/4.txt" "folder.rc") with temptree(files) as t: folder_texts = utils.search_tree(t, "folder*", ".txt") self.assertEquals(len(folder_texts), 4) f1 = os.path.join(t, "folder_file.txt") f2 = os.path.join(t, "folder/1.txt") f3 = os.path.join(t, "folder/sub/2.txt") f4 = os.path.join(t, "folder2/3.txt") for f in [f1, f2, f3, f4]: self.assert_(f in folder_texts)
def test_pattern_upload_all_logs(self): # test empty dir with temptree([]) as t: conf = {'log_dir': t} uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.run_once) def get_random_length_str(max_len=10, chars=string.ascii_letters): return ''.join(random.choice(chars) for x in range(random.randint(1, max_len))) template = 'prefix_%(random)s_%(digits)s.blah.' \ '%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz' pattern = '''prefix_.*_[0-9]+\.blah\. (?P<year>[0-9]{4}) (?P<month>[0-1][0-9]) (?P<day>[0-3][0-9]) (?P<hour>[0-2][0-9])00-[0-9]{2}00 -[0-9]?[0-9]\.gz''' files_that_should_match = [] # add some files that match for i in range(24): fname = template % { 'random': get_random_length_str(), 'digits': get_random_length_str(16, string.digits), 'datestr': datetime.now().strftime('%Y%m%d'), 'hour': i, 'next_hour': i + 1, 'number': random.randint(0, 20), } files_that_should_match.append(fname) # add some files that don't match files = list(files_that_should_match) for i in range(24): fname = template % { 'random': get_random_length_str(), 'digits': get_random_length_str(16, string.digits), 'datestr': datetime.now().strftime('%Y%m'), 'hour': i, 'next_hour': i + 1, 'number': random.randint(0, 20), } files.append(fname) for fname in files: print fname with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: self.assertEquals(len(os.listdir(t)), 48) conf = {'source_filename_pattern': pattern, 'log_dir': t} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(os.listdir(t)), 24) self.assertEquals(len(uploader.uploaded_files), 24) files_that_were_uploaded = set(x[0] for x in uploader.uploaded_files) for f in files_that_should_match: self.assert_(os.path.join(t, f) in files_that_were_uploaded)
def test_pattern_upload_all_logs(self): # test empty dir with temptree([]) as t: conf = {'log_dir': t} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 0) def get_random_length_str(max_len=10, chars=string.ascii_letters): return ''.join( random.choice(chars) for x in range(random.randint(1, max_len))) template = 'prefix_%(random)s_%(digits)s.blah.' \ '%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz' pattern = r'prefix_.*_[0-9]+\.blah\.%Y%m%d%H00-[0-9]{2}00' \ '-[0-9]?[0-9]\.gz' files_that_should_match = [] # add some files that match for i in range(24): fname = template % { 'random': get_random_length_str(), 'digits': get_random_length_str(16, string.digits), 'datestr': datetime.now().strftime('%Y%m%d'), 'hour': i, 'next_hour': i + 1, 'number': random.randint(0, 20), } files_that_should_match.append(fname) # add some files that don't match files = list(files_that_should_match) for i in range(24): fname = template % { 'random': get_random_length_str(), 'digits': get_random_length_str(16, string.digits), 'datestr': datetime.now().strftime('%Y%m'), 'hour': i, 'next_hour': i + 1, 'number': random.randint(0, 20), } files.append(fname) for fname in files: print fname with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: self.assertEquals(len(os.listdir(t)), 48) conf = {'source_filename_pattern': pattern, 'log_dir': t} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(os.listdir(t)), 24) self.assertEquals(len(uploader.uploaded_files), 24) files_that_were_uploaded = set(x[0] for x in uploader.uploaded_files) for f in files_that_should_match: self.assert_(os.path.join(t, f) in files_that_were_uploaded)
def test_search_tree(self): # file match & ext miss with temptree(['asdf.conf', 'blarg.conf', 'asdf.cfg']) as t: asdf = utils.search_tree(t, 'a*', '.conf') self.assertEquals(len(asdf), 1) self.assertEquals(asdf[0], os.path.join(t, 'asdf.conf')) # multi-file match & glob miss & sort with temptree(['application.bin', 'apple.bin', 'apropos.bin']) as t: app_bins = utils.search_tree(t, 'app*', 'bin') self.assertEquals(len(app_bins), 2) self.assertEquals(app_bins[0], os.path.join(t, 'apple.bin')) self.assertEquals(app_bins[1], os.path.join(t, 'application.bin')) # test file in folder & ext miss & glob miss files = ( 'sub/file1.ini', 'sub/file2.conf', 'sub.bin', 'bus.ini', 'bus/file3.ini', ) with temptree(files) as t: sub_ini = utils.search_tree(t, 'sub*', '.ini') self.assertEquals(len(sub_ini), 1) self.assertEquals(sub_ini[0], os.path.join(t, 'sub/file1.ini')) # test multi-file in folder & sub-folder & ext miss & glob miss files = ( 'folder_file.txt', 'folder/1.txt', 'folder/sub/2.txt', 'folder2/3.txt', 'Folder3/4.txt' 'folder.rc', ) with temptree(files) as t: folder_texts = utils.search_tree(t, 'folder*', '.txt') self.assertEquals(len(folder_texts), 4) f1 = os.path.join(t, 'folder_file.txt') f2 = os.path.join(t, 'folder/1.txt') f3 = os.path.join(t, 'folder/sub/2.txt') f4 = os.path.join(t, 'folder2/3.txt') for f in [f1, f2, f3, f4]: self.assert_(f in folder_texts)
def test_create_container_fail(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files) as t: conf = {'log_dir': t} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1) with temptree(files) as t: conf = {'log_dir': t} uploader = MockLogUploader(conf) # mock create_container to fail uploader.internal_proxy.create_container = lambda *args: False uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 0)
def test_two_realms_and_change_a_default(self): fname = 'container-sync-realms.conf' fcontents = ''' [DEFAULT] mtime_check_interval = 60 [US] key = 9ff3b71c849749dbaec4ccdd3cbab62b cluster_dfw1 = http://dfw1.host/v1/ [UK] key = e9569809dc8b4951accc1487aa788012 key2 = f6351bd1cc36413baa43f7ba1b45e51d cluster_lon3 = http://lon3.host/v1/ ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.all_log_lines(), {}) self.assertEqual(csr.mtime_check_interval, 60) self.assertEqual(sorted(csr.realms()), ['UK', 'US']) self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b') self.assertIsNone(csr.key2('US')) self.assertEqual(csr.clusters('US'), ['DFW1']) self.assertEqual(csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/') self.assertEqual(csr.key('UK'), 'e9569809dc8b4951accc1487aa788012') self.assertEqual(csr.key2('UK'), 'f6351bd1cc36413baa43f7ba1b45e51d') self.assertEqual(csr.clusters('UK'), ['LON3']) self.assertEqual(csr.endpoint('UK', 'LON3'), 'http://lon3.host/v1/')
def _proxy_modify_wsgi_pipeline(self, pipe): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = %s [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:catch_errors] use = egg:swift#catch_errors [filter:gatekeeper] use = egg:swift#gatekeeper """ config = config % (pipe,) contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) return app
def test_appconfig_dir_ignores_hidden_files(self): config_dir = { 'server.conf.d/01.conf': """ [app:main] use = egg:swift#proxy port = 8080 """, 'server.conf.d/.01.conf.swp': """ [app:main] use = egg:swift#proxy port = 8081 """, } # strip indent from test config contents config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) with temptree(*zip(*config_dir.items())) as path: conf_dir = os.path.join(path, 'server.conf.d') conf = wsgi.appconfig(conf_dir) expected = { '__file__': os.path.join(path, 'server.conf.d'), 'here': os.path.join(path, 'server.conf.d'), 'port': '8080', } self.assertEquals(conf, expected)
def test_init_request_processor(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = catch_errors proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors """ contents = dedent(config) with temptree(["proxy-server.conf"]) as t: conf_file = os.path.join(t, "proxy-server.conf") with open(conf_file, "w") as f: f.write(contents.replace("TEMPDIR", t)) _fake_rings(t) app, conf, logger, log_name = wsgi.init_request_processor(conf_file, "proxy-server") # verify pipeline is catch_errors -> proxy-servery expected = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assert_(isinstance(app, expected)) self.assert_(isinstance(app.app, swift.proxy.server.Application)) # config settings applied to app instance self.assertEquals(0.2, app.app.conn_timeout) # appconfig returns values from 'proxy-server' section expected = {"__file__": conf_file, "here": os.path.dirname(conf_file), "conn_timeout": "0.2", "swift_dir": t} self.assertEquals(expected, conf) # logger works logger.info("testing") self.assertEquals("proxy-server", log_name)
def test_two_realms_and_change_a_default(self): fname = "container-sync-realms.conf" fcontents = """ [DEFAULT] mtime_check_interval = 60 [US] key = 9ff3b71c849749dbaec4ccdd3cbab62b cluster_dfw1 = http://dfw1.host/v1/ [UK] key = e9569809dc8b4951accc1487aa788012 key2 = f6351bd1cc36413baa43f7ba1b45e51d cluster_lon3 = http://lon3.host/v1/ """ with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.lines_dict, {}) self.assertEqual(csr.mtime_check_interval, 60) self.assertEqual(sorted(csr.realms()), ["UK", "US"]) self.assertEqual(csr.key("US"), "9ff3b71c849749dbaec4ccdd3cbab62b") self.assertEqual(csr.key2("US"), None) self.assertEqual(csr.clusters("US"), ["DFW1"]) self.assertEqual(csr.endpoint("US", "DFW1"), "http://dfw1.host/v1/") self.assertEqual(csr.key("UK"), "e9569809dc8b4951accc1487aa788012") self.assertEqual(csr.key2("UK"), "f6351bd1cc36413baa43f7ba1b45e51d") self.assertEqual(csr.clusters("UK"), ["LON3"]) self.assertEqual(csr.endpoint("UK", "LON3"), "http://lon3.host/v1/")
def test_two_realms_and_change_a_default(self): fname = 'container-sync-realms.conf' fcontents = ''' [DEFAULT] mtime_check_interval = 60 [US] key = 9ff3b71c849749dbaec4ccdd3cbab62b cluster_dfw1 = http://dfw1.host/v1/ [UK] key = e9569809dc8b4951accc1487aa788012 key2 = f6351bd1cc36413baa43f7ba1b45e51d cluster_lon3 = http://lon3.host/v1/ ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.all_log_lines(), {}) self.assertEqual(csr.mtime_check_interval, 60) self.assertEqual(sorted(csr.realms()), ['UK', 'US']) self.assertEqual(csr.key('US'), '9ff3b71c849749dbaec4ccdd3cbab62b') self.assertEqual(csr.key2('US'), None) self.assertEqual(csr.clusters('US'), ['DFW1']) self.assertEqual( csr.endpoint('US', 'DFW1'), 'http://dfw1.host/v1/') self.assertEqual(csr.key('UK'), 'e9569809dc8b4951accc1487aa788012') self.assertEqual( csr.key2('UK'), 'f6351bd1cc36413baa43f7ba1b45e51d') self.assertEqual(csr.clusters('UK'), ['LON3']) self.assertEqual( csr.endpoint('UK', 'LON3'), 'http://lon3.host/v1/')
def setUp(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck catch_errors tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors [filter:healthcheck] use = egg:swift#healthcheck [filter:tempurl] paste.filter_factory = swift.common.middleware.tempurl:filter_factory """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) ctx = wsgi.loadcontext(loadwsgi.APP, conf_file, global_conf={}) self.pipe = wsgi.PipelineWrapper(ctx)
def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-logging bulk tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:proxy-logging] use = egg:swift#proxy_logging [filter:bulk] use = egg:swift#bulk [filter:tempurl] use = egg:swift#tempurl """ new_req_filters = [ # not in pipeline, no afters { 'name': 'catch_errors' }, # already in pipeline { 'name': 'proxy_logging', 'after_fn': lambda _: ['catch_errors'] }, # not in pipeline, comes after more than one thing { 'name': 'container_quotas', 'after_fn': lambda _: ['catch_errors', 'bulk'] } ] contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch.object(swift.proxy.server, 'required_filters', new_req_filters): app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.catch_errors', 'swift.common.middleware.healthcheck', 'swift.common.middleware.proxy_logging', 'swift.common.middleware.bulk', 'swift.common.middleware.container_quotas', 'swift.common.middleware.tempurl', 'swift.proxy.server' ])
def test_proxy_unmodified_wsgi_pipeline(self): # Make sure things are sane even when we modify nothing config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = catch_errors gatekeeper proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors [filter:gatekeeper] use = egg:swift#gatekeeper """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.proxy.server'])
def test_run_server_with_latest_eventlet(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = proxy-server [app:proxy-server] use = egg:swift#proxy """ def argspec_stub(server): return mock.MagicMock(args=["capitalize_response_headers"]) contents = dedent(config) with temptree(["proxy-server.conf"]) as t: conf_file = os.path.join(t, "proxy-server.conf") with open(conf_file, "w") as f: f.write(contents.replace("TEMPDIR", t)) _fake_rings(t) with nested( mock.patch("swift.proxy.server.Application." "modify_wsgi_pipeline"), mock.patch("swift.common.wsgi.wsgi"), mock.patch("swift.common.wsgi.eventlet"), mock.patch("swift.common.wsgi.inspect", getargspec=argspec_stub), ) as (_, _wsgi, _, _): conf = wsgi.appconfig(conf_file) logger = logging.getLogger("test") sock = listen(("localhost", 0)) wsgi.run_server(conf, logger, sock) _wsgi.server.assert_called() args, kwargs = _wsgi.server.call_args self.assertEquals(kwargs.get("capitalize_response_headers"), False)
def test_proxy_modify_wsgi_pipeline(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), ['swift.common.middleware.catch_errors', 'swift.common.middleware.gatekeeper', 'swift.common.middleware.healthcheck', 'swift.proxy.server'])
def test_bad_pattern_in_config(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: # invalid pattern conf = { 'log_dir': t, 'source_filename_pattern': '%Y%m%d%h' } # should be %H uploader = MockLogUploader(conf) self.assertFalse(uploader.validate_filename_pattern()) uploader.upload_all_logs() self.assertEquals(uploader.uploaded_files, []) conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%H'} uploader = MockLogUploader(conf) self.assert_(uploader.validate_filename_pattern()) uploader.upload_all_logs() self.assertEquals(len(uploader.uploaded_files), 1) # deprecated warning on source_filename_format class MockLogger(): def __init__(self): self.msgs = defaultdict(list) def log(self, level, msg): self.msgs[level].append(msg) def __getattr__(self, attr): return partial(self.log, attr) logger = MockLogger.logger = MockLogger() def mock_get_logger(*args, **kwargs): return MockLogger.logger _orig_get_logger = log_uploader.utils.get_logger try: log_uploader.utils.get_logger = mock_get_logger conf = {'source_filename_format': '%Y%m%d%H'} uploader = MockLogUploader(conf, logger=logger) self.assert_( [m for m in logger.msgs['warning'] if 'deprecated' in m]) finally: log_uploader.utils.get_logger = _orig_get_logger # convert source_filename_format to regex conf = {'source_filename_format': 'pattern-*.%Y%m%d%H.*.gz'} uploader = MockLogUploader(conf) expected = r'pattern-.*\.%Y%m%d%H\..*\.gz' self.assertEquals(uploader.pattern, expected) # use source_filename_pattern if we have the choice! conf = { 'source_filename_format': 'bad', 'source_filename_pattern': 'good', } uploader = MockLogUploader(conf) self.assertEquals(uploader.pattern, 'good')
def test_bad_pattern_in_config(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: # invalid pattern conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%h'} # should be %H uploader = MockLogUploader(conf) self.assertFalse(uploader.validate_filename_pattern()) uploader.upload_all_logs() self.assertEquals(uploader.uploaded_files, []) conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%H'} uploader = MockLogUploader(conf) self.assert_(uploader.validate_filename_pattern()) uploader.upload_all_logs() self.assertEquals(len(uploader.uploaded_files), 1) # deprecated warning on source_filename_format class MockLogger(): def __init__(self): self.msgs = defaultdict(list) def log(self, level, msg): self.msgs[level].append(msg) def __getattr__(self, attr): return partial(self.log, attr) logger = MockLogger.logger = MockLogger() def mock_get_logger(*args, **kwargs): return MockLogger.logger _orig_get_logger = log_uploader.utils.get_logger try: log_uploader.utils.get_logger = mock_get_logger conf = {'source_filename_format': '%Y%m%d%H'} uploader = MockLogUploader(conf, logger=logger) self.assert_([m for m in logger.msgs['warning'] if 'deprecated' in m]) finally: log_uploader.utils.get_logger = _orig_get_logger # convert source_filename_format to regex conf = {'source_filename_format': 'pattern-*.%Y%m%d%H.*.gz'} uploader = MockLogUploader(conf) expected = r'pattern-.*\.%Y%m%d%H\..*\.gz' self.assertEquals(uploader.pattern, expected) # use source_filename_pattern if we have the choice! conf = { 'source_filename_format': 'bad', 'source_filename_pattern': 'good', } uploader = MockLogUploader(conf) self.assertEquals(uploader.pattern, 'good')
def test_load_app(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:catch_errors] use = egg:swift#catch_errors [filter:healthcheck] use = egg:swift#healthcheck """ def modify_func(app, pipe): new = pipe.create_filter('catch_errors') pipe.insert_filter(new) contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch( 'swift.proxy.server.Application.modify_wsgi_pipeline', modify_func): app = wsgi.loadapp(conf_file, global_conf={}) exp = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assertTrue(isinstance(app, exp), app) exp = swift.common.middleware.healthcheck.HealthCheckMiddleware self.assertTrue(isinstance(app.app, exp), app.app) exp = swift.proxy.server.Application self.assertTrue(isinstance(app.app.app, exp), app.app.app) # make sure you can turn off the pipeline modification if you want def blow_up(*_, **__): raise self.fail("needs more struts") with mock.patch( 'swift.proxy.server.Application.modify_wsgi_pipeline', blow_up): app = wsgi.loadapp(conf_file, global_conf={}, allow_modify_pipeline=False) # the pipeline was untouched exp = swift.common.middleware.healthcheck.HealthCheckMiddleware self.assertTrue(isinstance(app, exp), app) exp = swift.proxy.server.Application self.assertTrue(isinstance(app.app, exp), app.app)
def test_empty(self): fname = "container-sync-realms.conf" fcontents = "" with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.lines_dict, {}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), [])
def test_empty(self): fname = 'container-sync-realms.conf' fcontents = '' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.all_log_lines(), {}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), [])
def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-logging bulk tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:proxy-logging] use = egg:swift#proxy_logging [filter:bulk] use = egg:swift#bulk [filter:tempurl] use = egg:swift#tempurl """ new_req_filters = [ # not in pipeline, no afters {"name": "catch_errors"}, # already in pipeline {"name": "proxy_logging", "after_fn": lambda _: ["catch_errors"]}, # not in pipeline, comes after more than one thing {"name": "container_quotas", "after_fn": lambda _: ["catch_errors", "bulk"]}, ] contents = dedent(config) with temptree(["proxy-server.conf"]) as t: conf_file = os.path.join(t, "proxy-server.conf") with open(conf_file, "w") as f: f.write(contents.replace("TEMPDIR", t)) _fake_rings(t) with mock.patch.object(swift.proxy.server, "required_filters", new_req_filters): app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual( self.pipeline_modules(app), [ "swift.common.middleware.catch_errors", "swift.common.middleware.healthcheck", "swift.common.middleware.proxy_logging", "swift.common.middleware.bulk", "swift.common.middleware.container_quotas", "swift.common.middleware.tempurl", "swift.proxy.server", ], )
def test_log_cutoff(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files) as t: conf = {'log_dir': t, 'new_log_cutoff': '7200'} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 0) conf = {'log_dir': t, 'new_log_cutoff': '0'} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1)
def test_remove_file(self): with temptree([]) as t: file_name = os.path.join(t, 'blah.pid') # assert no raise self.assertEquals(os.path.exists(file_name), False) self.assertEquals(utils.remove_file(file_name), None) with open(file_name, 'w') as f: f.write('1') self.assert_(os.path.exists(file_name)) self.assertEquals(utils.remove_file(file_name), None) self.assertFalse(os.path.exists(file_name))
def test_get_sig(self): fname = 'container-sync-realms.conf' fcontents = '' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( csr.get_sig('GET', '/some/path', '1387212345.67890', 'my_nonce', 'realm_key', 'user_key'), '5a6eb486eb7b44ae1b1f014187a94529c3f9c8f9')
def test_get_sig(self): fname = "container-sync-realms.conf" fcontents = "" with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( csr.get_sig("GET", "/some/path", "1387212345.67890", "my_nonce", "realm_key", "user_key"), "5a6eb486eb7b44ae1b1f014187a94529c3f9c8f9", )
def temp_config(contents): fd, fn = mkstemp() with temptree(['proxy-server.conf']) as t: conf_dir = os.path.join(t, 'proxy-server.conf.d') os.mkdir(conf_dir) conf_file = os.path.join(conf_dir, 'default.conf') with open(conf_file, 'w') as f: f.write(contents) f.close() loader = wsgi.ConfigDirLoader(conf_dir) yield loader.parser
def test_proxy_modify_wsgi_pipeline_ordering(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = healthcheck proxy-logging bulk tempurl proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 [filter:healthcheck] use = egg:swift#healthcheck [filter:proxy-logging] use = egg:swift#proxy_logging [filter:bulk] use = egg:swift#bulk [filter:tempurl] use = egg:swift#tempurl """ new_req_filters = [ # not in pipeline, no afters {'name': 'catch_errors'}, # already in pipeline {'name': 'proxy_logging', 'after': ['catch_errors']}, # not in pipeline, comes after more than one thing {'name': 'container_quotas', 'after': ['catch_errors', 'bulk']}] contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch.object(swift.proxy.server, 'required_filters', new_req_filters): app = wsgi.loadapp(conf_file, global_conf={}) self.assertEqual(self.pipeline_modules(app), [ 'swift.common.middleware.catch_errors', 'swift.common.middleware.healthcheck', 'swift.common.middleware.proxy_logging', 'swift.common.middleware.bulk', 'swift.common.middleware.container_quotas', 'swift.common.middleware.tempurl', 'swift.proxy.server'])
def test_get_sig(self): fname = 'container-sync-realms.conf' fcontents = '' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( csr.get_sig( 'GET', '/some/path', '1387212345.67890', 'my_nonce', 'realm_key', 'user_key'), '5a6eb486eb7b44ae1b1f014187a94529c3f9c8f9')
def test_run_server_conf_dir(self): config_dir = { 'proxy-server.conf.d/pipeline.conf': """ [pipeline:main] pipeline = proxy-server """, 'proxy-server.conf.d/app.conf': """ [app:proxy-server] use = egg:swift#proxy """, 'proxy-server.conf.d/default.conf': """ [DEFAULT] eventlet_debug = yes client_timeout = 30 """ } # strip indent from test config contents config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) with temptree(*zip(*config_dir.items())) as conf_root: conf_dir = os.path.join(conf_root, 'proxy-server.conf.d') with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f: f.write('[DEFAULT]\nswift_dir = %s' % conf_root) _fake_rings(conf_root) with mock.patch('swift.proxy.server.Application.' 'modify_wsgi_pipeline'): with mock.patch('swift.common.wsgi.wsgi') as _wsgi: with mock.patch('swift.common.wsgi.eventlet') as _eventlet: with mock.patch.dict('os.environ', {'TZ': ''}): with mock.patch('swift.common.wsgi.inspect'): conf = wsgi.appconfig(conf_dir) logger = logging.getLogger('test') sock = listen(('localhost', 0)) wsgi.run_server(conf, logger, sock) self.assert_(os.environ['TZ'] is not '') self.assertEquals('HTTP/1.0', _wsgi.HttpProtocol.default_request_version) self.assertEquals(30, _wsgi.WRITE_TIMEOUT) _eventlet.hubs.use_hub.assert_called_with(utils.get_hub()) _eventlet.patcher.monkey_patch.assert_called_with(all=False, socket=True) _eventlet.debug.hub_exceptions.assert_called_with(True) _wsgi.server.assert_called() args, kwargs = _wsgi.server.call_args server_sock, server_app, server_logger = args self.assertEquals(sock, server_sock) self.assert_(isinstance(server_app, swift.proxy.server.Application)) self.assert_(isinstance(server_logger, wsgi.NullLogger)) self.assert_('custom_pool' in kwargs)
def test_run_server_debug(self): config = """ [DEFAULT] eventlet_debug = yes client_timeout = 30 max_clients = 1000 swift_dir = TEMPDIR [pipeline:main] pipeline = proxy-server [app:proxy-server] use = egg:swift#proxy # while "set" values normally override default set client_timeout = 20 # this section is not in conf during run_server set max_clients = 10 """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) with mock.patch('swift.proxy.server.Application.' 'modify_wsgi_pipeline'): with mock.patch('swift.common.wsgi.wsgi') as _wsgi: mock_server = _wsgi.server _wsgi.server = lambda *args, **kwargs: mock_server( *args, **kwargs) with mock.patch('swift.common.wsgi.eventlet') as _eventlet: conf = wsgi.appconfig(conf_file) logger = logging.getLogger('test') sock = listen(('localhost', 0)) wsgi.run_server(conf, logger, sock) self.assertEquals('HTTP/1.0', _wsgi.HttpProtocol.default_request_version) self.assertEquals(30, _wsgi.WRITE_TIMEOUT) _eventlet.hubs.use_hub.assert_called_with(utils.get_hub()) _eventlet.patcher.monkey_patch.assert_called_with(all=False, socket=True) _eventlet.debug.hub_exceptions.assert_called_with(True) self.assertTrue(mock_server.called) args, kwargs = mock_server.call_args server_sock, server_app, server_logger = args self.assertEquals(sock, server_sock) self.assert_(isinstance(server_app, swift.proxy.server.Application)) self.assertEquals(20, server_app.client_timeout) self.assertEqual(server_logger, None) self.assert_('custom_pool' in kwargs) self.assertEquals(1000, kwargs['custom_pool'].size)
def test_bad_upload(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: # invalid pattern conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%h'} # should be %H uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.upload_all_logs) conf = {'log_dir': t, 'source_filename_pattern': access_regex} uploader = ErrorLogUploader(conf) # this tests if the exception is handled uploader.upload_all_logs()
def test_bad_pattern_in_config(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: # invalid pattern conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%h'} # should be %H uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.upload_all_logs) conf = {'log_dir': t, 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) uploader.upload_all_logs() self.assertEquals(len(uploader.uploaded_files), 1)
def test_error_parsing(self): fname = 'container-sync-realms.conf' fcontents = 'invalid' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( logger.lines_dict, {'error': [ "Could not load '%s': File contains no section headers.\n" "file: %s, line: 1\n" "'invalid'" % (fpath, fpath)]}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), [])
def test_upload_file_failed(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA]) as t: conf = {'log_dir': t, 'unlink_log': 'true'} uploader = MockLogUploader(conf) # mock upload_file to fail, and clean up mock def mock_upload_file(self, *args, **kwargs): uploader.uploaded_files.pop() return False uploader.internal_proxy.upload_file = mock_upload_file uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 0) # file still there self.assertEquals(len(os.listdir(t)), 1)
def test_error_parsing(self): fname = 'container-sync-realms.conf' fcontents = 'invalid' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( logger.all_log_lines(), {'error': [ "Could not load '%s': File contains no section headers.\n" "file: %s, line: 1\n" "'invalid'" % (fpath, fpath)]}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), [])
def test_upload_file_failed(self): files = ['plugin-%s' % datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA]) as t: conf = {'log_dir': t, 'unlink_log': 'true', 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) # mock upload_file to fail, and clean up mock def mock_upload_file(self, *args, **kwargs): uploader.uploaded_files.pop() return False uploader.internal_proxy.upload_file = mock_upload_file self.assertRaises(SystemExit, uploader.run_once) # file still there self.assertEquals(len(os.listdir(t)), 1)
def test_init_request_processor(self): config = """ [DEFAULT] swift_dir = TEMPDIR [pipeline:main] pipeline = proxy-server [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 """ contents = dedent(config) with temptree(['proxy-server.conf']) as t: conf_file = os.path.join(t, 'proxy-server.conf') with open(conf_file, 'w') as f: f.write(contents.replace('TEMPDIR', t)) _fake_rings(t) app, conf, logger, log_name = wsgi.init_request_processor( conf_file, 'proxy-server') # verify pipeline is catch_errors -> dlo -> proxy-server expected = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assert_(isinstance(app, expected)) app = app.app expected = swift.common.middleware.gatekeeper.GatekeeperMiddleware self.assert_(isinstance(app, expected)) app = app.app expected = swift.common.middleware.dlo.DynamicLargeObject self.assert_(isinstance(app, expected)) app = app.app expected = swift.proxy.server.Application self.assert_(isinstance(app, expected)) # config settings applied to app instance self.assertEquals(0.2, app.conn_timeout) # appconfig returns values from 'proxy-server' section expected = { '__file__': conf_file, 'here': os.path.dirname(conf_file), 'conn_timeout': '0.2', 'swift_dir': t, } self.assertEquals(expected, conf) # logger works logger.info('testing') self.assertEquals('proxy-server', log_name)
def test_bad_mtime_check_interval(self): fname = 'container-sync-realms.conf' fcontents = ''' [DEFAULT] mtime_check_interval = invalid ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual( logger.lines_dict, {'error': [ "Error in '%s' with mtime_check_interval: invalid literal " "for int() with base 10: 'invalid'" % fpath]}) self.assertEqual(csr.mtime_check_interval, 300)
def test_write_file(self): with temptree([]) as t: file_name = os.path.join(t, 'test') utils.write_file(file_name, 'test') with open(file_name, 'r') as f: contents = f.read() self.assertEquals(contents, 'test') # and also subdirs file_name = os.path.join(t, 'subdir/test2') utils.write_file(file_name, 'test2') with open(file_name, 'r') as f: contents = f.read() self.assertEquals(contents, 'test2') # but can't over-write files file_name = os.path.join(t, 'subdir/test2/test3') self.assertRaises(IOError, utils.write_file, file_name, 'test3')
def test_unlink_log(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA]) as t: conf = {'log_dir': t, 'unlink_log': 'false'} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1) # file still there self.assertEquals(len(os.listdir(t)), 1) conf = {'log_dir': t, 'unlink_log': 'true'} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1) # file gone self.assertEquals(len(os.listdir(t)), 0)
def test_empty_realm(self): fname = "container-sync-realms.conf" fcontents = """ [US] """ with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.lines_dict, {}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), ["US"]) self.assertEqual(csr.key("US"), None) self.assertEqual(csr.key2("US"), None) self.assertEqual(csr.clusters("US"), []) self.assertEqual(csr.endpoint("US", "JUST_TESTING"), None)
def test_empty_realm(self): fname = 'container-sync-realms.conf' fcontents = ''' [US] ''' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) self.assertEqual(logger.all_log_lines(), {}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), ['US']) self.assertIsNone(csr.key('US')) self.assertIsNone(csr.key2('US')) self.assertEqual(csr.clusters('US'), []) self.assertIsNone(csr.endpoint('US', 'JUST_TESTING'))
def test_init_request_processor_from_conf_dir(self): config_dir = { 'proxy-server.conf.d/pipeline.conf': """ [pipeline:main] pipeline = catch_errors proxy-server """, 'proxy-server.conf.d/app.conf': """ [app:proxy-server] use = egg:swift#proxy conn_timeout = 0.2 """, 'proxy-server.conf.d/catch-errors.conf': """ [filter:catch_errors] use = egg:swift#catch_errors """ } # strip indent from test config contents config_dir = dict((f, dedent(c)) for (f, c) in config_dir.items()) with mock.patch('swift.proxy.server.Application.modify_wsgi_pipeline'): with temptree(*zip(*config_dir.items())) as conf_root: conf_dir = os.path.join(conf_root, 'proxy-server.conf.d') with open(os.path.join(conf_dir, 'swift.conf'), 'w') as f: f.write('[DEFAULT]\nswift_dir = %s' % conf_root) _fake_rings(conf_root) app, conf, logger, log_name = wsgi.init_request_processor( conf_dir, 'proxy-server') # verify pipeline is catch_errors -> proxy-server expected = swift.common.middleware.catch_errors.CatchErrorMiddleware self.assert_(isinstance(app, expected)) self.assert_(isinstance(app.app, swift.proxy.server.Application)) # config settings applied to app instance self.assertEquals(0.2, app.app.conn_timeout) # appconfig returns values from 'proxy-server' section expected = { '__file__': conf_dir, 'here': conf_dir, 'conn_timeout': '0.2', 'swift_dir': conf_root, } self.assertEquals(expected, conf) # logger works logger.info('testing') self.assertEquals('proxy-server', log_name)
def test_bad_mtime_check_interval(self): fname = 'container-sync-realms.conf' fcontents = ''' [DEFAULT] mtime_check_interval = invalid ''' with temptree([fname], [fcontents]) as tempdir: logger = debug_logger() fpath = os.path.join(tempdir, fname) csr = ContainerSyncRealms(fpath, logger) logs = logger.all_log_lines() self.assertEqual(logs, {'error': [ANY]}) line = logs['error'][0] self.assertIn( "Error in '%s' with mtime_check_interval: " "could not convert string to float:" % fpath, line) self.assertEqual(csr.mtime_check_interval, 300)
def test_os_error(self): fname = 'container-sync-realms.conf' fcontents = '' with temptree([fname], [fcontents]) as tempdir: logger = FakeLogger() fpath = os.path.join(tempdir, fname) os.chmod(tempdir, 0) csr = ContainerSyncRealms(fpath, logger) try: self.assertEqual( logger.lines_dict, {'error': [ "Could not load '%s': [Errno 13] Permission denied: " "'%s'" % (fpath, fpath)]}) self.assertEqual(csr.mtime_check_interval, 300) self.assertEqual(csr.realms(), []) finally: os.chmod(tempdir, 0700)