def test_pattern_upload_all_logs(self): # test empty dir with temptree([]) as t: conf = {'log_dir': t} uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.run_once) def get_random_length_str(max_len=10, chars=string.ascii_letters): return ''.join(random.choice(chars) for x in range(random.randint(1, max_len))) template = 'prefix_%(random)s_%(digits)s.blah.' \ '%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz' pattern = '''prefix_.*_[0-9]+\.blah\. (?P<year>[0-9]{4}) (?P<month>[0-1][0-9]) (?P<day>[0-3][0-9]) (?P<hour>[0-2][0-9])00-[0-9]{2}00 -[0-9]?[0-9]\.gz''' files_that_should_match = [] # add some files that match for i in range(24): fname = template % { 'random': get_random_length_str(), 'digits': get_random_length_str(16, string.digits), 'datestr': datetime.now().strftime('%Y%m%d'), 'hour': i, 'next_hour': i + 1, 'number': random.randint(0, 20), } files_that_should_match.append(fname) # add some files that don't match files = list(files_that_should_match) for i in range(24): fname = template % { 'random': get_random_length_str(), 'digits': get_random_length_str(16, string.digits), 'datestr': datetime.now().strftime('%Y%m'), 'hour': i, 'next_hour': i + 1, 'number': random.randint(0, 20), } files.append(fname) for fname in files: print fname with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: self.assertEquals(len(os.listdir(t)), 48) conf = {'source_filename_pattern': pattern, 'log_dir': t} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(os.listdir(t)), 24) self.assertEquals(len(uploader.uploaded_files), 24) files_that_were_uploaded = set(x[0] for x in uploader.uploaded_files) for f in files_that_should_match: self.assert_(os.path.join(t, f) in files_that_were_uploaded)
def test_create_container_fail(self): files = [datetime.now().strftime('%Y%m%d%H')] conf = {'source_filename_pattern': access_regex} with temptree(files) as t: conf['log_dir'] = t uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1) with temptree(files) as t: conf['log_dir'] = t uploader = MockLogUploader(conf) # mock create_container to fail uploader.internal_proxy.create_container = lambda *args: False uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 0)
def test_log_cutoff(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files) as t: conf = {'log_dir': t, 'new_log_cutoff': '7200', 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 0) conf = {'log_dir': t, 'new_log_cutoff': '0', 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1)
def test_bad_pattern_in_config(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: # invalid pattern conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%h'} # should be %H uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.upload_all_logs) conf = {'log_dir': t, 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) uploader.upload_all_logs() self.assertEquals(len(uploader.uploaded_files), 1)
def test_bad_upload(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: # invalid pattern conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%h'} # should be %H uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.upload_all_logs) conf = {'log_dir': t, 'source_filename_pattern': access_regex} uploader = ErrorLogUploader(conf) # this tests if the exception is handled uploader.upload_all_logs()
def test_upload_file_failed(self): files = ['plugin-%s' % datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA]) as t: conf = {'log_dir': t, 'unlink_log': 'true', 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) # mock upload_file to fail, and clean up mock def mock_upload_file(self, *args, **kwargs): uploader.uploaded_files.pop() return False uploader.internal_proxy.upload_file = mock_upload_file self.assertRaises(SystemExit, uploader.run_once) # file still there self.assertEquals(len(os.listdir(t)), 1)
def test_bad_upload(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: # invalid pattern conf = { 'log_dir': t, 'source_filename_pattern': '%Y%m%d%h' } # should be %H uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.upload_all_logs) conf = {'log_dir': t, 'source_filename_pattern': access_regex} uploader = ErrorLogUploader(conf) # this tests if the exception is handled uploader.upload_all_logs()
def test_bad_pattern_in_config(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: # invalid pattern conf = { 'log_dir': t, 'source_filename_pattern': '%Y%m%d%h' } # should be %H uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.upload_all_logs) conf = {'log_dir': t, 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) uploader.upload_all_logs() self.assertEquals(len(uploader.uploaded_files), 1)
def test_unlink_log(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA]) as t: conf = {'log_dir': t, 'unlink_log': 'false', 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1) # file still there self.assertEquals(len(os.listdir(t)), 1) conf = {'log_dir': t, 'unlink_log': 'true', 'source_filename_pattern': access_regex} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1) # file gone self.assertEquals(len(os.listdir(t)), 0)
def test_process_one_file(self): with temptree([]) as t: conf = {'working_dir': t, 'swift_account': 'foo', 'log_source_account': 'bar'} p = access_log_delivery.AccessLogDelivery(conf, DumbLogger()) def my_get_object_data(*a, **kw): all_lines = [] log_line = [str(x) for x in range(18)] log_line[1] = 'proxy-server' log_line[4] = '1/Jan/3/4/5/6' log_line[6] = '/v1/a/c/o?foo' yield 'x' * 16 + ' '.join(log_line) log_line = [str(x) for x in range(18)] log_line[1] = 'proxy-server' log_line[4] = '1/Jan/3/4/5/6' log_line[6] = '/v1/a/c/o' yield 'x' * 16 + ' '.join(log_line) log_line = [str(x) for x in range(18)] log_line[1] = 'proxy-server' log_line[4] = '1/Jan/3/4/5/6' log_line[6] = '/v1/a2/c2/o2' yield 'x' * 16 + ' '.join(log_line) def my_get_container_save_log_flag(*a, **kw): return True p.get_object_data = my_get_object_data p.get_container_save_log_flag = my_get_container_save_log_flag res = p.process_one_file('a', 'c', '2011/03/14/12/hash') expected = ['%s/a2/c2/2011/03/14/12' % t, '%s/a/c/2011/03/14/12' % t] self.assertEquals(res, set(expected)) lines = [p.convert_log_line(x)[0] for x in my_get_object_data()] with open(expected[0], 'rb') as f: raw = f.read() res = '\n'.join(lines[2:]) + '\n' self.assertEquals(res, raw) with open(expected[1], 'rb') as f: raw = f.read() res = '\n'.join(lines[:2]) + '\n' self.assertEquals(res, raw)
def test_upload_file_failed(self): files = ['plugin-%s' % datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA]) as t: conf = { 'log_dir': t, 'unlink_log': 'true', 'source_filename_pattern': access_regex } uploader = MockLogUploader(conf) # mock upload_file to fail, and clean up mock def mock_upload_file(self, *args, **kwargs): uploader.uploaded_files.pop() return False uploader.internal_proxy.upload_file = mock_upload_file self.assertRaises(SystemExit, uploader.run_once) # file still there self.assertEquals(len(os.listdir(t)), 1)
def test_log_cutoff(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files) as t: conf = { 'log_dir': t, 'new_log_cutoff': '7200', 'source_filename_pattern': access_regex } uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 0) conf = { 'log_dir': t, 'new_log_cutoff': '0', 'source_filename_pattern': access_regex } uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1)
def test_unlink_log(self): files = [datetime.now().strftime('%Y%m%d%H')] with temptree(files, contents=[COMPRESSED_DATA]) as t: conf = { 'log_dir': t, 'unlink_log': 'false', 'source_filename_pattern': access_regex } uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1) # file still there self.assertEquals(len(os.listdir(t)), 1) conf = { 'log_dir': t, 'unlink_log': 'true', 'source_filename_pattern': access_regex } uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(uploader.uploaded_files), 1) # file gone self.assertEquals(len(os.listdir(t)), 0)
def test_pattern_upload_all_logs(self): # test empty dir with temptree([]) as t: conf = {'log_dir': t} uploader = MockLogUploader(conf) self.assertRaises(SystemExit, uploader.run_once) def get_random_length_str(max_len=10, chars=string.ascii_letters): return ''.join( random.choice(chars) for x in range(random.randint(1, max_len))) template = 'prefix_%(random)s_%(digits)s.blah.' \ '%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz' pattern = '''prefix_.*_[0-9]+\.blah\. (?P<year>[0-9]{4}) (?P<month>[0-1][0-9]) (?P<day>[0-3][0-9]) (?P<hour>[0-2][0-9])00-[0-9]{2}00 -[0-9]?[0-9]\.gz''' files_that_should_match = [] # add some files that match for i in range(24): fname = template % { 'random': get_random_length_str(), 'digits': get_random_length_str(16, string.digits), 'datestr': datetime.now().strftime('%Y%m%d'), 'hour': i, 'next_hour': i + 1, 'number': random.randint(0, 20), } files_that_should_match.append(fname) # add some files that don't match files = list(files_that_should_match) for i in range(24): fname = template % { 'random': get_random_length_str(), 'digits': get_random_length_str(16, string.digits), 'datestr': datetime.now().strftime('%Y%m'), 'hour': i, 'next_hour': i + 1, 'number': random.randint(0, 20), } files.append(fname) for fname in files: print fname with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: self.assertEquals(len(os.listdir(t)), 48) conf = {'source_filename_pattern': pattern, 'log_dir': t} uploader = MockLogUploader(conf) uploader.run_once() self.assertEquals(len(os.listdir(t)), 24) self.assertEquals(len(uploader.uploaded_files), 24) files_that_were_uploaded = set(x[0] for x in uploader.uploaded_files) for f in files_that_should_match: self.assert_(os.path.join(t, f) in files_that_were_uploaded)