def test_atomic_write_dict_as_json_file(self):
info = {'a': "hi"}
scalyr_util.atomic_write_dict_as_json_file(self.__path,
self.__path + '~', info)
json_object = scalyr_util.read_file_as_json(self.__path)
self.assertEquals(json_object, JsonObject(a='hi'))
def test_atomic_write_dict_as_json_file(self):
info = {"a": "hi"}
scalyr_util.atomic_write_dict_as_json_file(self.__path,
self.__path + "~", info)
json_object = scalyr_util.read_file_as_json(self.__path)
self.assertEquals(json_object, info)
def __update_checkpoints( self ):
# updatedate the api's checkpoints
self.__api.update_checkpoints()
# save to disk
if self.__api.checkpoints:
tmp_file = self.__checkpoint_file + '~'
scalyr_util.atomic_write_dict_as_json_file( self.__checkpoint_file, tmp_file, self.__api.checkpoints )
def __update_checkpoints( self ):
# updatedate the api's checkpoints
self.__api.update_checkpoints()
# save to disk
if self.__api.checkpoints:
tmp_file = self.__checkpoint_file + '~'
scalyr_util.atomic_write_dict_as_json_file( self.__checkpoint_file, tmp_file, self.__api.checkpoints )
def write_checkpoint_state_to_file(checkpoints, file_path, current_time):
"""
Write the the checkpoints collection to file.
"""
tmp_path = file_path + "~"
state = {
"time": current_time,
"checkpoints": checkpoints,
}
# We write to a temporary file and then rename it to the real file name to make the write more atomic.
# We have had problems in the past with corrupted checkpoint files due to failures during the write.
scalyr_util.atomic_write_dict_as_json_file(file_path, tmp_path, state)
def __update_checkpoints( self ):
"""Update the checkpoints for when each docker logger logged a request, and save the checkpoints
to file.
"""
for logger in self.docker_loggers:
last_request = logger.last_request()
self.__checkpoints[logger.stream_name] = last_request
# save to disk
if self.__checkpoints:
tmp_file = self.__checkpoint_file + '~'
scalyr_util.atomic_write_dict_as_json_file( self.__checkpoint_file, tmp_file, self.__checkpoints )
def update_checkpoint(self, name, value):
"""
Update the value of the named checkpoint, and write the checkpoints to disk as JSON
@param name: the name of the checkpoint
@param value: the value to set the checkpoint to
"""
self._lock.acquire()
try:
self._checkpoints[name] = value
tmp_file = self._filename + "~"
scalyr_util.atomic_write_dict_as_json_file(self._filename,
tmp_file,
self._checkpoints)
finally:
self._lock.release()
def __update_checkpoints(self):
"""Update the checkpoints for when each docker logger logged a request, and save the checkpoints
to file.
"""
for logger in self.docker_loggers:
last_request = logger.last_request()
self.__checkpoints[logger.stream_name] = last_request
# save to disk
if self.__checkpoints:
tmp_file = self.__checkpoint_file + '~'
scalyr_util.atomic_write_dict_as_json_file(self.__checkpoint_file,
tmp_file,
self.__checkpoints)
def write_checkpoint_state_to_file(checkpoints, file_path, current_time):
# type: (Dict, six.text_type, float) -> None
"""
Write checkpoint state into file.
:param checkpoints: The checkpoint state stored in dict.
:param file_path: Output file path.
"""
# We write to a temporary file and then rename it to the real file name to make the write more atomic.
# We have had problems in the past with corrupted checkpoint files due to failures during the write.
tmp_path = file_path + "~"
state = {
"time": current_time,
"checkpoints": checkpoints,
}
scalyr_util.atomic_write_dict_as_json_file(file_path, tmp_path, state)
def __write_checkpoint_state(self):
"""Writes the current checkpoint state to disk.
This must be done periodically to ensure that if the agent process stops and starts up again, we pick up
from where we left off copying each file.
"""
# Create the format that is expected. An overall JsonObject with the time when the file was written,
# and then an entry for each file path.
checkpoints = {}
state = {
'time': time.time(),
'checkpoints': checkpoints,
}
for processor in self.__log_processors:
checkpoints[processor.log_path] = processor.get_checkpoint()
# We write to a temporary file and then rename it to the real file name to make the write more atomic.
# We have had problems in the past with corrupted checkpoint files due to failures during the write.
file_path = os.path.join(self.__config.agent_data_path, 'checkpoints.json')
tmp_path = os.path.join(self.__config.agent_data_path, 'checkpoints.json~')
scalyr_util.atomic_write_dict_as_json_file( file_path, tmp_path, state )
def test_atomic_write_dict_as_json_file_error(self):
# mock of the 'atomic_write_dict_as_json_file' internal funtion calls to raise an error
# and validate the error log message.
info = {"a": "hi"}
with patch("os.rename") as os_rename_mock:
os_rename_mock.side_effect = Exception("I am an error.")
scalyr_util.atomic_write_dict_as_json_file(self.__path,
self.__temp_file_path,
info)
self.assertLogFileContainsLineRegex(expression="I am an error.")
self.assertLogFileContainsLineRegex(
expression="File path: '{0}', type: {1}".format(
re.escape(self.__path), type(self.__path)))
self.assertLogFileContainsLineRegex(
expression="Temporary file path: '{0}', type: {1}".format(
re.escape(self.__temp_file_path), type(self.__temp_file_path)))
self.assertLogFileContainsLineRegex(expression="File exists: False.")
self.assertLogFileContainsLineRegex(
expression="Temporary file exists: True.")
self.assertLogFileContainsLineRegex(
expression="File system encoding: {0}".format(
re.escape(sys.getfilesystemencoding())))
def __write_checkpoint_state(self):
"""Writes the current checkpoint state to disk.
This must be done periodically to ensure that if the agent process stops and starts up again, we pick up
from where we left off copying each file.
"""
# Create the format that is expected. An overall JsonObject with the time when the file was written,
# and then an entry for each file path.
checkpoints = {}
state = {
'time': time.time(),
'checkpoints': checkpoints,
}
for processor in self.__log_processors:
checkpoints[processor.log_path] = processor.get_checkpoint()
# We write to a temporary file and then rename it to the real file name to make the write more atomic.
# We have had problems in the past with corrupted checkpoint files due to failures during the write.
file_path = os.path.join(self.__config.agent_data_path,
'checkpoints.json')
tmp_path = os.path.join(self.__config.agent_data_path,
'checkpoints.json~')
scalyr_util.atomic_write_dict_as_json_file(file_path, tmp_path, state)
def test_atomic_write_dict_as_json_file(self):
info = { 'a': "hi" }
scalyr_util.atomic_write_dict_as_json_file( self.__path, self.__path + '~', info )
json_object = scalyr_util.read_file_as_json( self.__path )
self.assertEquals( json_object, JsonObject( a='hi' ) )
