def put_data_to_local_log(self, log_name, token, data):
"""
The method pushes given data (which is a string) to the data queue for given log, which is identified by
lof_name, of the backend.
Before placing new data to the queue the method checks if there is no such log file named by log_name
value in the logs map; if there is no such log - creates it.
:param log_name - str - name of the log file without the path to it (to reference it in the logs map):
:param data - str - data to be pushed to the data queue:
:return:
"""
try:
try:
self.log_map_lock.acquire()
if data is None:
return
timestamp = TimeUtils.get_current_time_as_timestamp_as_ms()
if self.logs_map.get(
log_name
) is None: # New log - need to add it to the map
self.add_to_local_logs_map(log_name, token, timestamp)
else:
# First message timestamp is set to None after log rotation.
if self.logs_map[log_name]['first_msg_ts'] is None:
self.logs_map[log_name]['first_msg_ts'] = timestamp
finally:
self.log_map_lock.release()
if not self.no_timestamps:
data = str(timestamp) + ' ' + data
data_size = len(data)
new_data_item = {
'log_name': log_name,
'token': token,
'data': data,
'size': data_size,
'timestamp': timestamp
}
while len(self.data_queue) == self.data_queue.maxlen:
time.sleep(DATA_QUEUE_FREE_WAIT_PERIOD)
self.data_queue.appendleft(new_data_item)
self.data_queue_thread.data_consumer_event.set()
except threading.ThreadError, e:
log.error('Cannot acquire log write lock! Error %s' % e.message)
raise
def put_data_to_local_log(self, log_name, token, data):
"""
The method pushes given data (which is a string) to the data queue for given log, which is identified by
lof_name, of the backend.
Before placing new data to the queue the method checks if there is no such log file named by log_name
value in the logs map; if there is no such log - creates it.
:param log_name - str - name of the log file without the path to it (to reference it in the logs map):
:param data - str - data to be pushed to the data queue:
:return:
"""
try:
try:
self.log_map_lock.acquire()
if data is None:
return
timestamp = TimeUtils.get_current_time_as_timestamp_as_ms()
if self.logs_map.get(log_name) is None: # New log - need to add it to the map
self.add_to_local_logs_map(log_name, token, timestamp)
else:
# First message timestamp is set to None after log rotation.
if self.logs_map[log_name]['first_msg_ts'] is None:
self.logs_map[log_name]['first_msg_ts'] = timestamp
finally:
self.log_map_lock.release()
if not self.no_timestamps:
data = str(timestamp) + ' ' + data
data_size = len(data)
new_data_item = {'log_name': log_name, 'token': token, 'data': data, 'size': data_size,
'timestamp': timestamp}
while len(self.data_queue) == self.data_queue.maxlen:
time.sleep(DATA_QUEUE_FREE_WAIT_PERIOD)
self.data_queue.appendleft(new_data_item)
self.data_queue_thread.data_consumer_event.set()
except threading.ThreadError, e:
log.error('Cannot acquire log write lock! Error %s' % e.message)
raise
def rotate_log(self, log_object, log_name):
"""
The method performs log rotation which consists of next operations:
1) The method tries to get the first timestamp for the given log object (which is stored in 'first_msg_ts' field);
2) If it is None by some reason, the method uses current timestamp with ms. precision to generate the name;
3) If such log exists - the method tries to generate new name, expanding the old one with '_INDEX' (1,2,3,...)
4) After new name is generated the method renames given *.log file with new generated name (e.g. 1234567890.log)
5) After renaming the method adjusts log object's info: 1 - sets total written data size to 0; 2 - sets first
log's timestamp to None (this is completely new file - it does not contain timestamps yet); 3 - alters last
message timestamp field with last_data_timestamp value - it is needed by check_rotation_needed() method on
next iteration of new message processing for this log object.
6) Pushes new name of the rotated file into LogsArchiver's queue and signalizes that the file needs to be
compressed.
:param log_object - dict - log object from logs_map:
:param log_name - str - the name of currently rotated log (like 'amazon_s3_XXXXX.log'):
:return rotation_success - boolean - signalizes whether log rotation succeeded:
"""
old_file_name = log_object['local_log_file']
rotation_success = False
new_file_name = ARCHIVING_BACKEND_BASE_DIRECTORY + log_object[
'token'] + '/'
if not os.path.exists(new_file_name):
os.makedirs(new_file_name)
# Generate new name for the rotated log from it's first backend timestamp or from the
# current timestamp (current date-time) if it is absent by some reason, like: 212347772784.log
if log_object['first_msg_ts'] is None:
numeric_name_part = str(
TimeUtils.get_current_time_as_timestamp_as_ms())
new_file_name += numeric_name_part + '.log'
else:
numeric_name_part = str(log_object['first_msg_ts'])
new_file_name += numeric_name_part + '.log'
# Check whether we have any files with the name the same as we've just generated. If there are ones -
# generate new name by adding "_N" prefix to it (N=1,2,3...)
if os.path.exists(new_file_name):
generated_name = ''
for i in range(1, MAX_FILE_NAME_INDEX + 1):
generated_name = ARCHIVING_BACKEND_BASE_DIRECTORY + log_object[
'token'] + '/' + numeric_name_part + '_' + str(i) + '.log'
if not os.path.exists(generated_name):
break
if i == MAX_FILE_NAME_INDEX:
raise Exception(
'Cannot generate new rotation name: all indexes from _%d to _%d '
'are already used!' % (1, MAX_FILE_NAME_INDEX))
log.info(
'Rotation: %s already exists, so %s name will be used for the log rotation.'
% (new_file_name, generated_name))
new_file_name = generated_name
try:
try:
self.log_map_lock.acquire()
os.rename(old_file_name, new_file_name)
log_object[
'size'] = 0 # Size of written data is cleared due to the file rotation.
log_object[
'first_msg_ts'] = None # The same is for the first message's timestamp.
self.logs_map[log_name] = log_object
rotation_success = True
if not self.no_logs_compressing:
self.logs_compressor.compress_async(
new_file_name, self.compress_callback)
except threading.ThreadError, e:
log.error('Cannot acquire log map lock! Error %s' % e.message)
except Exception, e:
if hasattr(e, 'strerror'):
msg = e.strerror # For IO errors and other system-related ones
else:
msg = e.message
log.error(
'Cannot rename %s to %s for log rotation. Error: %s' %
(old_file_name, new_file_name, msg))
def rotate_log(self, log_object, log_name):
"""
The method performs log rotation which consists of next operations:
1) The method tries to get the first timestamp for the given log object (which is stored in 'first_msg_ts' field);
2) If it is None by some reason, the method uses current timestamp with ms. precision to generate the name;
3) If such log exists - the method tries to generate new name, expanding the old one with '_INDEX' (1,2,3,...)
4) After new name is generated the method renames given *.log file with new generated name (e.g. 1234567890.log)
5) After renaming the method adjusts log object's info: 1 - sets total written data size to 0; 2 - sets first
log's timestamp to None (this is completely new file - it does not contain timestamps yet); 3 - alters last
message timestamp field with last_data_timestamp value - it is needed by check_rotation_needed() method on
next iteration of new message processing for this log object.
6) Pushes new name of the rotated file into LogsArchiver's queue and signalizes that the file needs to be
compressed.
:param log_object - dict - log object from logs_map:
:param log_name - str - the name of currently rotated log (like 'amazon_s3_XXXXX.log'):
:return rotation_success - boolean - signalizes whether log rotation succeeded:
"""
old_file_name = log_object['local_log_file']
rotation_success = False
new_file_name = ARCHIVING_BACKEND_BASE_DIRECTORY + log_object['token'] + '/'
if not os.path.exists(new_file_name):
os.makedirs(new_file_name)
# Generate new name for the rotated log from it's first backend timestamp or from the
# current timestamp (current date-time) if it is absent by some reason, like: 212347772784.log
if log_object['first_msg_ts'] is None:
numeric_name_part = str(TimeUtils.get_current_time_as_timestamp_as_ms())
new_file_name += numeric_name_part + '.log'
else:
numeric_name_part = str(log_object['first_msg_ts'])
new_file_name += numeric_name_part + '.log'
# Check whether we have any files with the name the same as we've just generated. If there are ones -
# generate new name by adding "_N" prefix to it (N=1,2,3...)
if os.path.exists(new_file_name):
generated_name = ''
for i in range(1, MAX_FILE_NAME_INDEX + 1):
generated_name = ARCHIVING_BACKEND_BASE_DIRECTORY + log_object['token'] + '/' + numeric_name_part + '_' + str(i) + '.log'
if not os.path.exists(generated_name):
break
if i == MAX_FILE_NAME_INDEX:
raise Exception('Cannot generate new rotation name: all indexes from _%d to _%d '
'are already used!' % (1, MAX_FILE_NAME_INDEX))
log.info('Rotation: %s already exists, so %s name will be used for the log rotation.' %
(new_file_name, generated_name))
new_file_name = generated_name
try:
try:
self.log_map_lock.acquire()
os.rename(old_file_name, new_file_name)
log_object['size'] = 0 # Size of written data is cleared due to the file rotation.
log_object['first_msg_ts'] = None # The same is for the first message's timestamp.
self.logs_map[log_name] = log_object
rotation_success = True
if not self.no_logs_compressing:
self.logs_compressor.compress_async(new_file_name, self.compress_callback)
except threading.ThreadError, e:
log.error('Cannot acquire log map lock! Error %s' % e.message)
except Exception, e:
if hasattr(e, 'strerror'):
msg = e.strerror # For IO errors and other system-related ones
else:
msg = e.message
log.error('Cannot rename %s to %s for log rotation. Error: %s' % (old_file_name, new_file_name, msg))
