def test_pattern_upload_all_logs(self):
# test empty dir
with temptree([]) as t:
conf = {'log_dir': t}
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.run_once)
def get_random_length_str(max_len=10, chars=string.ascii_letters):
return ''.join(random.choice(chars) for x in
range(random.randint(1, max_len)))
template = 'prefix_%(random)s_%(digits)s.blah.' \
'%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz'
pattern = '''prefix_.*_[0-9]+\.blah\.
(?P<year>[0-9]{4})
(?P<month>[0-1][0-9])
(?P<day>[0-3][0-9])
(?P<hour>[0-2][0-9])00-[0-9]{2}00
-[0-9]?[0-9]\.gz'''
files_that_should_match = []
# add some files that match
for i in range(24):
fname = template % {
'random': get_random_length_str(),
'digits': get_random_length_str(16, string.digits),
'datestr': datetime.now().strftime('%Y%m%d'),
'hour': i,
'next_hour': i + 1,
'number': random.randint(0, 20),
}
files_that_should_match.append(fname)
# add some files that don't match
files = list(files_that_should_match)
for i in range(24):
fname = template % {
'random': get_random_length_str(),
'digits': get_random_length_str(16, string.digits),
'datestr': datetime.now().strftime('%Y%m'),
'hour': i,
'next_hour': i + 1,
'number': random.randint(0, 20),
}
files.append(fname)
for fname in files:
print fname
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
self.assertEquals(len(os.listdir(t)), 48)
conf = {'source_filename_pattern': pattern, 'log_dir': t}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(os.listdir(t)), 24)
self.assertEquals(len(uploader.uploaded_files), 24)
files_that_were_uploaded = set(x[0] for x in
uploader.uploaded_files)
for f in files_that_should_match:
self.assert_(os.path.join(t, f) in files_that_were_uploaded)
def test_create_container_fail(self):
files = [datetime.now().strftime('%Y%m%d%H')]
conf = {'source_filename_pattern': access_regex}
with temptree(files) as t:
conf['log_dir'] = t
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
with temptree(files) as t:
conf['log_dir'] = t
uploader = MockLogUploader(conf)
# mock create_container to fail
uploader.internal_proxy.create_container = lambda *args: False
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
def test_create_container_fail(self):
files = [datetime.now().strftime('%Y%m%d%H')]
conf = {'source_filename_pattern': access_regex}
with temptree(files) as t:
conf['log_dir'] = t
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
with temptree(files) as t:
conf['log_dir'] = t
uploader = MockLogUploader(conf)
# mock create_container to fail
uploader.internal_proxy.create_container = lambda *args: False
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
def test_log_cutoff(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files) as t:
conf = {'log_dir': t, 'new_log_cutoff': '7200',
'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
conf = {'log_dir': t, 'new_log_cutoff': '0',
'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
def test_bad_pattern_in_config(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
# invalid pattern
conf = {'log_dir': t,
'source_filename_pattern': '%Y%m%d%h'} # should be %H
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.upload_all_logs)
conf = {'log_dir': t, 'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
uploader.upload_all_logs()
self.assertEquals(len(uploader.uploaded_files), 1)
def test_bad_upload(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
# invalid pattern
conf = {'log_dir': t,
'source_filename_pattern': '%Y%m%d%h'} # should be %H
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.upload_all_logs)
conf = {'log_dir': t, 'source_filename_pattern': access_regex}
uploader = ErrorLogUploader(conf)
# this tests if the exception is handled
uploader.upload_all_logs()
def test_upload_file_failed(self):
files = ['plugin-%s' % datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA]) as t:
conf = {'log_dir': t, 'unlink_log': 'true',
'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
# mock upload_file to fail, and clean up mock
def mock_upload_file(self, *args, **kwargs):
uploader.uploaded_files.pop()
return False
uploader.internal_proxy.upload_file = mock_upload_file
self.assertRaises(SystemExit, uploader.run_once)
# file still there
self.assertEquals(len(os.listdir(t)), 1)
def test_bad_upload(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
# invalid pattern
conf = {
'log_dir': t,
'source_filename_pattern': '%Y%m%d%h'
} # should be %H
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.upload_all_logs)
conf = {'log_dir': t, 'source_filename_pattern': access_regex}
uploader = ErrorLogUploader(conf)
# this tests if the exception is handled
uploader.upload_all_logs()
def test_bad_pattern_in_config(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
# invalid pattern
conf = {
'log_dir': t,
'source_filename_pattern': '%Y%m%d%h'
} # should be %H
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.upload_all_logs)
conf = {'log_dir': t, 'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
uploader.upload_all_logs()
self.assertEquals(len(uploader.uploaded_files), 1)
def test_unlink_log(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA]) as t:
conf = {'log_dir': t, 'unlink_log': 'false',
'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
# file still there
self.assertEquals(len(os.listdir(t)), 1)
conf = {'log_dir': t, 'unlink_log': 'true',
'source_filename_pattern': access_regex}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
# file gone
self.assertEquals(len(os.listdir(t)), 0)
def test_process_one_file(self):
with temptree([]) as t:
conf = {'working_dir': t, 'swift_account': 'foo',
'log_source_account': 'bar'}
p = access_log_delivery.AccessLogDelivery(conf, DumbLogger())
def my_get_object_data(*a, **kw):
all_lines = []
log_line = [str(x) for x in range(18)]
log_line[1] = 'proxy-server'
log_line[4] = '1/Jan/3/4/5/6'
log_line[6] = '/v1/a/c/o?foo'
yield 'x' * 16 + ' '.join(log_line)
log_line = [str(x) for x in range(18)]
log_line[1] = 'proxy-server'
log_line[4] = '1/Jan/3/4/5/6'
log_line[6] = '/v1/a/c/o'
yield 'x' * 16 + ' '.join(log_line)
log_line = [str(x) for x in range(18)]
log_line[1] = 'proxy-server'
log_line[4] = '1/Jan/3/4/5/6'
log_line[6] = '/v1/a2/c2/o2'
yield 'x' * 16 + ' '.join(log_line)
def my_get_container_save_log_flag(*a, **kw):
return True
p.get_object_data = my_get_object_data
p.get_container_save_log_flag = my_get_container_save_log_flag
res = p.process_one_file('a', 'c', '2011/03/14/12/hash')
expected = ['%s/a2/c2/2011/03/14/12' % t,
'%s/a/c/2011/03/14/12' % t]
self.assertEquals(res, set(expected))
lines = [p.convert_log_line(x)[0] for x in my_get_object_data()]
with open(expected[0], 'rb') as f:
raw = f.read()
res = '\n'.join(lines[2:]) + '\n'
self.assertEquals(res, raw)
with open(expected[1], 'rb') as f:
raw = f.read()
res = '\n'.join(lines[:2]) + '\n'
self.assertEquals(res, raw)
def test_upload_file_failed(self):
files = ['plugin-%s' % datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA]) as t:
conf = {
'log_dir': t,
'unlink_log': 'true',
'source_filename_pattern': access_regex
}
uploader = MockLogUploader(conf)
# mock upload_file to fail, and clean up mock
def mock_upload_file(self, *args, **kwargs):
uploader.uploaded_files.pop()
return False
uploader.internal_proxy.upload_file = mock_upload_file
self.assertRaises(SystemExit, uploader.run_once)
# file still there
self.assertEquals(len(os.listdir(t)), 1)
def test_log_cutoff(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files) as t:
conf = {
'log_dir': t,
'new_log_cutoff': '7200',
'source_filename_pattern': access_regex
}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 0)
conf = {
'log_dir': t,
'new_log_cutoff': '0',
'source_filename_pattern': access_regex
}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
def test_process_one_file(self):
with temptree([]) as t:
conf = {'working_dir': t, 'swift_account': 'foo',
'log_source_account': 'bar'}
p = access_log_delivery.AccessLogDelivery(conf, DumbLogger())
def my_get_object_data(*a, **kw):
all_lines = []
log_line = [str(x) for x in range(18)]
log_line[1] = 'proxy-server'
log_line[4] = '1/Jan/3/4/5/6'
log_line[6] = '/v1/a/c/o?foo'
yield 'x' * 16 + ' '.join(log_line)
log_line = [str(x) for x in range(18)]
log_line[1] = 'proxy-server'
log_line[4] = '1/Jan/3/4/5/6'
log_line[6] = '/v1/a/c/o'
yield 'x' * 16 + ' '.join(log_line)
log_line = [str(x) for x in range(18)]
log_line[1] = 'proxy-server'
log_line[4] = '1/Jan/3/4/5/6'
log_line[6] = '/v1/a2/c2/o2'
yield 'x' * 16 + ' '.join(log_line)
def my_get_container_save_log_flag(*a, **kw):
return True
p.get_object_data = my_get_object_data
p.get_container_save_log_flag = my_get_container_save_log_flag
res = p.process_one_file('a', 'c', '2011/03/14/12/hash')
expected = ['%s/a2/c2/2011/03/14/12' % t,
'%s/a/c/2011/03/14/12' % t]
self.assertEquals(res, set(expected))
lines = [p.convert_log_line(x)[0] for x in my_get_object_data()]
with open(expected[0], 'rb') as f:
raw = f.read()
res = '\n'.join(lines[2:]) + '\n'
self.assertEquals(res, raw)
with open(expected[1], 'rb') as f:
raw = f.read()
res = '\n'.join(lines[:2]) + '\n'
self.assertEquals(res, raw)
def test_unlink_log(self):
files = [datetime.now().strftime('%Y%m%d%H')]
with temptree(files, contents=[COMPRESSED_DATA]) as t:
conf = {
'log_dir': t,
'unlink_log': 'false',
'source_filename_pattern': access_regex
}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
# file still there
self.assertEquals(len(os.listdir(t)), 1)
conf = {
'log_dir': t,
'unlink_log': 'true',
'source_filename_pattern': access_regex
}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(uploader.uploaded_files), 1)
# file gone
self.assertEquals(len(os.listdir(t)), 0)
def test_pattern_upload_all_logs(self):
# test empty dir
with temptree([]) as t:
conf = {'log_dir': t}
uploader = MockLogUploader(conf)
self.assertRaises(SystemExit, uploader.run_once)
def get_random_length_str(max_len=10, chars=string.ascii_letters):
return ''.join(
random.choice(chars)
for x in range(random.randint(1, max_len)))
template = 'prefix_%(random)s_%(digits)s.blah.' \
'%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz'
pattern = '''prefix_.*_[0-9]+\.blah\.
(?P<year>[0-9]{4})
(?P<month>[0-1][0-9])
(?P<day>[0-3][0-9])
(?P<hour>[0-2][0-9])00-[0-9]{2}00
-[0-9]?[0-9]\.gz'''
files_that_should_match = []
# add some files that match
for i in range(24):
fname = template % {
'random': get_random_length_str(),
'digits': get_random_length_str(16, string.digits),
'datestr': datetime.now().strftime('%Y%m%d'),
'hour': i,
'next_hour': i + 1,
'number': random.randint(0, 20),
}
files_that_should_match.append(fname)
# add some files that don't match
files = list(files_that_should_match)
for i in range(24):
fname = template % {
'random': get_random_length_str(),
'digits': get_random_length_str(16, string.digits),
'datestr': datetime.now().strftime('%Y%m'),
'hour': i,
'next_hour': i + 1,
'number': random.randint(0, 20),
}
files.append(fname)
for fname in files:
print fname
with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t:
self.assertEquals(len(os.listdir(t)), 48)
conf = {'source_filename_pattern': pattern, 'log_dir': t}
uploader = MockLogUploader(conf)
uploader.run_once()
self.assertEquals(len(os.listdir(t)), 24)
self.assertEquals(len(uploader.uploaded_files), 24)
files_that_were_uploaded = set(x[0]
for x in uploader.uploaded_files)
for f in files_that_should_match:
self.assert_(os.path.join(t, f) in files_that_were_uploaded)
