def rotate_log_files(options):
with request_lock(options['lock_file']) as acquired:
if not acquired:
logger.warn('Not rotating, previous job still underway')
return
# Check we can send signals to all relevant processes
pids_for_processes = running_processes_by_name(options['reopen_file_signals'].keys())
unkillable_processes = set()
for process_name in options['reopen_file_signals'].keys():
pids = pids_for_processes[process_name]
try:
for pid in pids:
kill_if_running(pid, 0)
except OSError:
unkillable_processes.add(process_name)
if unkillable_processes:
logger.error('Cannot send signal to some processes, aborting: %s' % ', '.join(unkillable_processes))
return
files_to_rotate = [
file for file in os.listdir(options['log_directory'])
if fnmatch.fnmatch(file, options['filename_filter'])
]
rotation_suffix = datetime.datetime.now().strftime(options['timestamp_format'])
filename_mapping = {
file: file + rotation_suffix
for file in files_to_rotate
}
# Move all files
rotated_files = []
for original_name, rotated_name in filename_mapping.items():
original_path = os.path.join(options['log_directory'], original_name)
rotated_path = os.path.join(options['log_directory'], rotated_name)
if not os.path.exists(rotated_path):
os.rename(original_path, rotated_path)
rotated_files.append(rotated_name)
else:
logger.warning('Did not rotate file. File called %s already existed', rotated_path)
# Run kick commands
pids_for_processes = running_processes_by_name(options['reopen_file_signals'].keys())
for process_name, signal_name in options['reopen_file_signals'].items():
signal_id = getattr(signal, 'SIG' + signal_name.upper())
pids = pids_for_processes[process_name]
for pid in pids:
kill_if_running(pid, signal_id)
throttle_file_checks = Throttle(FILE_OPEN_CHECK_INTERVAL)
checks_without_closed_files = 0
s3_store = S3LogStore(options)
# Get files which have no open handles and process them as soon as we can.
# Files with open handles wait until next time through the loop. We throttle
# to avoid checking too often.
# TODO: Should we also pick up and retry copying any gz files which we could not
# copy to s3 last time around?
open_files = rotated_files
while open_files:
throttle_file_checks.wait()
closed_files, open_files = check_for_open_files(open_files)
for ready_file in closed_files:
try:
ready_path = os.path.join(options['log_directory'], ready_file)
compressed_path = compress_file(ready_path)
s3_store.store_file(compressed_path)
os.unlink(compressed_path)
except:
logger.error('Unexpected error processing %s', ready_file, exc_info=True)
if len(closed_files):
checks_without_closed_files = 0
else:
checks_without_closed_files += 1
if checks_without_closed_files > MAX_CHECKS_WITHOUT_FILE_CLOSED:
logger.error('Gave up waiting for files to close. Open files: %s' % ', '.join(open_files))
return
def test_should_return_empty_list_when_no_process_running(self):
with running_process('/bin/cat'):
running_processes = running_processes_by_name(
['there_is_no_process_with_this_name'])
self.assertNotIn('there_is_no_process_with_this_name',
running_processes)
def rotate_log_files(options):
with request_lock(options['lock_file']) as acquired:
if not acquired:
logger.warn('Not rotating, previous job still underway')
return
# Check we can send signals to all relevant processes
pids_for_processes = running_processes_by_name(
options['reopen_file_signals'].keys())
unkillable_processes = set()
for process_name in options['reopen_file_signals'].keys():
pids = pids_for_processes[process_name]
try:
for pid in pids:
kill_if_running(pid, 0)
except OSError:
unkillable_processes.add(process_name)
if unkillable_processes:
logger.error('Cannot send signal to some processes, aborting: %s' %
', '.join(unkillable_processes))
return
files_to_rotate = [
file for file in os.listdir(options['log_directory'])
if fnmatch.fnmatch(file, options['filename_filter'])
]
rotation_suffix = datetime.datetime.now().strftime(
options['timestamp_format'])
filename_mapping = {
file: file + rotation_suffix
for file in files_to_rotate
}
# Move all files
rotated_files = []
for original_name, rotated_name in filename_mapping.items():
original_path = os.path.join(options['log_directory'],
original_name)
rotated_path = os.path.join(options['log_directory'], rotated_name)
if not os.path.exists(rotated_path):
os.rename(original_path, rotated_path)
rotated_files.append(rotated_name)
else:
logger.warning(
'Did not rotate file. File called %s already existed',
rotated_path)
# Run kick commands
pids_for_processes = running_processes_by_name(
options['reopen_file_signals'].keys())
for process_name, signal_name in options['reopen_file_signals'].items(
):
signal_id = getattr(signal, 'SIG' + signal_name.upper())
pids = pids_for_processes[process_name]
for pid in pids:
kill_if_running(pid, signal_id)
throttle_file_checks = Throttle(FILE_OPEN_CHECK_INTERVAL)
checks_without_closed_files = 0
s3_store = S3LogStore(options)
# Get files which have no open handles and process them as soon as we can.
# Files with open handles wait until next time through the loop. We throttle
# to avoid checking too often.
# TODO: Should we also pick up and retry copying any gz files which we could not
# copy to s3 last time around?
open_files = rotated_files
while open_files:
throttle_file_checks.wait()
closed_files, open_files = check_for_open_files(open_files)
for ready_file in closed_files:
try:
ready_path = os.path.join(options['log_directory'],
ready_file)
compressed_path = compress_file(ready_path)
s3_store.store_file(compressed_path)
os.unlink(compressed_path)
except:
logger.error('Unexpected error processing %s',
ready_file,
exc_info=True)
if len(closed_files):
checks_without_closed_files = 0
else:
checks_without_closed_files += 1
if checks_without_closed_files > MAX_CHECKS_WITHOUT_FILE_CLOSED:
logger.error(
'Gave up waiting for files to close. Open files: %s' %
', '.join(open_files))
return
def test_should_handle_multiple_processes_same_name(self):
with running_process('/bin/cat') as pid:
with running_process('/bin/cat') as pid2:
running_processes = running_processes_by_name(['cat'])
self.assertIn(pid, running_processes['cat'])
self.assertIn(pid2, running_processes['cat'])
def test_should_handle_processes_with_arguments(self):
with running_process(['/bin/cat', 'foo', '-']) as pid:
running_processes = running_processes_by_name(['cat'])
self.assertIn(pid, running_processes['cat'])
def test_should_identify_running_process(self):
with running_process('/bin/cat') as pid:
running_processes = running_processes_by_name(['cat'])
self.assertIn(pid, running_processes['cat'])
def test_should_return_empty_list_when_no_process_running(self):
with running_process('/bin/cat'):
running_processes = running_processes_by_name(['there_is_no_process_with_this_name'])
self.assertNotIn('there_is_no_process_with_this_name', running_processes)
def test_should_handle_processes_with_arguments(self):
with running_process(['/bin/cat', 'foo', '-']) as pid:
running_processes = running_processes_by_name(['cat'])
self.assertIn(pid, running_processes['cat'])
def test_should_handle_multiple_processes_same_name(self):
with running_process('/bin/cat') as pid:
with running_process('/bin/cat') as pid2:
running_processes = running_processes_by_name(['cat'])
self.assertIn(pid, running_processes['cat'])
self.assertIn(pid2, running_processes['cat'])
def test_should_identify_running_process(self):
with running_process('/bin/cat') as pid:
running_processes = running_processes_by_name(['cat'])
self.assertIn(pid, running_processes['cat'])