def _calculate_per_thread_extra_bytes():
"""Calculates how many extra bytes are added to the serialized form of the threads JSON array
when adding a new thread, excluding the bytes for serializing the thread id and name themselves.
This is used below by the PostFixBuffer abstraction to help calculate the number of bytes the serialized form
of the PostFixBuffer will take, without having to actually serialize it. It was found that doing the heavy
weight process of serializing it over and over again to just get the size was eating too much CPU.
@return: An array of two int entries. The first entry is how many extra bytes are added when adding the
first thread to the threads JSON array and the second is how many extra bytes are added for all subsequent
threads. (The number differences by at least one due to the need for a comma to be inserted).
@rtype: [int]
"""
# An array of the number of bytes used to serialize the array when there are N threads in it (where N is the
# index into size_by_entries).
sizes_by_entries = []
# Calculate sizes_by_entries by actually serialzing each case.
threads = []
test_string = 'A'
for i in range(3):
sizes_by_entries.append(len(json_lib.serialize(threads)))
# Add in another thread for the next round through the loop.
threads.append({'id': test_string, 'name': test_string})
# Now go back and calculate the deltas between the different cases. We have to remember to subtract
# out the length due to the id and name strings.
test_string_len = len(json_lib.serialize(test_string))
result = []
for i in range(1, 3):
result.append(sizes_by_entries[i] - sizes_by_entries[i - 1] -
2 * test_string_len)
return result
def test_str_conversion(self):
self.assertEquals(self.get(1, convert_to=str), '1')
self.assertEquals(self.get('ah', convert_to=str), 'ah')
self.assertEquals(self.get(False, convert_to=str), 'False')
self.assertEquals(self.get(1.3, convert_to=str), '1.3')
self.assertEquals(self.get(1L, convert_to=str), '1')
test_array = ["a", "b", "c"]
# str -> ArrayOfStrings (must support different variations)
arr = ArrayOfStrings(*test_array)
self.assertEquals(self.get('a,b,c', convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get('a,b, c', convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get('"a", "b", "c"', convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get("'a', 'b', 'c'", convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get("[a, b, c]" , convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get("['a', \"b\", c]" , convert_to=ArrayOfStrings), arr)
# str -> JsonArray
self.assertEquals(self.get(serialize(test_array), convert_to=JsonArray), JsonArray(*test_array))
with self.assertRaises(BadMonitorConfiguration):
# single quotes are invalid JSON
self.assertEquals(self.get(str(test_array), convert_to=JsonArray), JsonArray(*test_array))
# str -> JsonObject
test_obj = {'a': 1, 'b': 'two', 'c': [1, 2, 3]}
self.assertEquals(self.get(serialize(test_obj), convert_to=JsonObject), json_lib_parse(serialize(test_obj)))
def add_event(self, event, timestamp=None):
"""Adds the serialized JSON for event if it does not cause the maximum request size to be exceeded.
It will automatically add in a 'ts' field to event containing a new timestamp based on the current time
but ensuring it is greater than any previous timestamp that has been used.
It is illegal to invoke this method if 'get_payload' has already been invoked.
@param event: The event object, usually a dict or a JsonObject.
@param timestamp: The timestamp to use for the event. This should only be used for testing.
@return: True if the event's serialized JSON was added to the request, or False if that would have resulted
in the maximum request size being exceeded so it did not.
"""
start_pos = self.__buffer.tell()
# If we already added an event before us, then make sure we add in a comma to separate us from the last event.
if self.__events_added > 0:
self.__buffer.write(',')
if timestamp is None:
timestamp = self.__get_timestamp()
event['ts'] = str(timestamp)
json_lib.serialize(event, output=self.__buffer, use_fast_encoding=True)
# Check if we exceeded the size, if so chop off what we just added.
if self.__current_size > self.__max_size:
self.__buffer.truncate(start_pos)
return False
self.__events_added += 1
return True
def test_str_conversion(self):
self.assertEquals(self.get(1, convert_to=str), '1')
self.assertEquals(self.get('ah', convert_to=str), 'ah')
self.assertEquals(self.get(False, convert_to=str), 'False')
self.assertEquals(self.get(1.3, convert_to=str), '1.3')
self.assertEquals(self.get(1L, convert_to=str), '1')
test_array = ["a", "b", "c"]
# str -> ArrayOfStrings (must support different variations)
arr = ArrayOfStrings(*test_array)
self.assertEquals(self.get('a,b,c', convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get('a,b, c', convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get('"a", "b", "c"', convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get("'a', 'b', 'c'", convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get("[a, b, c]" , convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get("['a', \"b\", c]" , convert_to=ArrayOfStrings), arr)
# str -> JsonArray
self.assertEquals(self.get(serialize(test_array), convert_to=JsonArray), JsonArray(*test_array))
self.assertRaises(
BadMonitorConfiguration,
# single quotes are invalid JSON
lambda: self.assertEquals(self.get(str(test_array), convert_to=JsonArray), JsonArray(*test_array)))
# str -> JsonObject
test_obj = {'a': 1, 'b': 'two', 'c': [1, 2, 3]}
self.assertEquals(self.get(serialize(test_obj), convert_to=JsonObject), json_lib_parse(serialize(test_obj)))
def add_event(self, event, timestamp=None):
"""Adds the serialized JSON for event if it does not cause the maximum request size to be exceeded.
It will automatically add in a 'ts' field to event containing a new timestamp based on the current time
but ensuring it is greater than any previous timestamp that has been used.
It is illegal to invoke this method if 'get_payload' has already been invoked.
@param event: The event object, usually a dict or a JsonObject.
@param timestamp: The timestamp to use for the event. This should only be used for testing.
@return: True if the event's serialized JSON was added to the request, or False if that would have resulted
in the maximum request size being exceeded so it did not.
"""
start_pos = self.__buffer.tell()
# If we already added an event before us, then make sure we add in a comma to separate us from the last event.
if self.__events_added > 0:
self.__buffer.write(',')
if timestamp is None:
timestamp = self.__get_timestamp()
event['ts'] = str(timestamp)
json_lib.serialize(event, output=self.__buffer, use_fast_encoding=True)
size = self.__buffer.tell() - start_pos
# Check if we exceeded the size, if so chop off what we just added.
if self.__current_size + size > self.__max_size:
self.__buffer.truncate(start_pos)
return False
self.__current_size += size
self.__events_added += 1
return True
def test_4byte_utf8(self):
actual = '\xF0\xAA\x9A\xA5'
expected_fast = '"\xf0\xaa\\u009a\xa5"'
expected_slow = '"\\u2a6a5"'
self.assertEquals(serialize(actual, use_fast_encoding=True),
expected_fast)
self.assertEquals(serialize(actual, use_fast_encoding=False),
expected_slow)
def test_4byte_utf8_slow(self):
actual = '\xF0\xAA\x9A\xA5'
expected_slow = '"\\U0002a6a5"'
self.assertEquals(serialize(actual, use_fast_encoding=False), expected_slow)
actual = '\xF0\x9F\x98\xA2'
expected_slow = '"\\U0001f622"'
self.assertEquals(serialize(actual, use_fast_encoding=False), expected_slow)
def test_4byte_utf8_fast(self):
actual = '\xF0\xAA\x9A\xA5'
expected_fast = '"\xf0\xaa\\u009a\xa5"'
self.assertEquals(serialize(actual, use_fast_encoding=True), expected_fast)
actual = '\xF0\x9F\x98\xA2'
expected_fast = '"\xf0\\u009f\\u0098\xa2"'
self.assertEquals(serialize(actual, use_fast_encoding=True), expected_fast)
def test_4byte_utf8_fast(self):
actual = '\xF0\xAA\x9A\xA5'
expected_fast = '"\xf0\xaa\\u009a\xa5"'
self.assertEquals(serialize(actual, use_fast_encoding=True),
expected_fast)
actual = '\xF0\x9F\x98\xA2'
expected_fast = '"\xf0\\u009f\\u0098\xa2"'
self.assertEquals(serialize(actual, use_fast_encoding=True),
expected_fast)
def test_4byte_utf8_slow(self):
actual = '\xF0\xAA\x9A\xA5'
expected_slow = '"\\U0002a6a5"'
self.assertEquals(serialize(actual, use_fast_encoding=False),
expected_slow)
actual = '\xF0\x9F\x98\xA2'
expected_slow = '"\\U0001f622"'
self.assertEquals(serialize(actual, use_fast_encoding=False),
expected_slow)
def atomic_write_dict_as_json_file(file_path, tmp_path, info):
"""Write a dict to a JSON encoded file
The file is first completely written to tmp_path, and then renamed to file_path
@param: file_path: The final path of the file
@param: tmp_path: A temporary path to write the file to
@param: info: A dict containing the JSON object to write
"""
fp = None
try:
fp = open(tmp_path, 'w')
fp.write(json_lib.serialize(info))
fp.close()
fp = None
if sys.platform == 'win32' and os.path.isfile(file_path):
os.unlink(file_path)
os.rename(tmp_path, file_path)
except (IOError, OSError):
if fp is not None:
fp.close()
import scalyr_agent.scalyr_logging
scalyr_agent.scalyr_logging.getLogger(__name__).exception(
'Could not write checkpoint file due to error',
error_code='failedCheckpointWrite')
def __create_test_instance(self, config_monitors, platform_monitors):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
monitors_json_array = JsonArray()
for entry in config_monitors:
monitors_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(
JsonObject(api_key='fake', monitors=monitors_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
return MonitorsManager(config, FakePlatform(platform_monitors))
def test_string(self):
self.__run_string_test_case('Hi there', '"Hi there"')
self.__run_string_test_case('Hi there\n', '"Hi there\\n"')
self.__run_string_test_case('Hi there\b', '"Hi there\\b"')
self.__run_string_test_case('Hi there\f', '"Hi there\\f"')
self.__run_string_test_case('Hi there\r', '"Hi there\\r"')
self.__run_string_test_case('Hi there\t', '"Hi there\\t"')
self.__run_string_test_case('Hi there\"', '"Hi there\\""')
self.__run_string_test_case('Hi there\\', '"Hi there\\\\"')
self.__run_string_test_case('Escaped\5', '"Escaped\\u0005"')
self.__run_string_test_case('Escaped\17', '"Escaped\\u000f"')
self.__run_string_test_case('Escaped\177', '"Escaped\\u007f"')
self.assertEquals(serialize('Escaped\xE2\x82\xAC', use_fast_encoding=True), '"Escaped\xe2\\u0082\xac"')
self.assertEquals(serialize('Escaped\xE2\x82\xAC', use_fast_encoding=False), '"Escaped\\u20ac"')
def create_configuration(extra_toplevel_config=None):
"""Creates a blank configuration file with default values. Optionally overwrites top-level key/values.
Sets api_key to 'fake' unless defined in extra_toplevel_config
@param extra_toplevel_config: Dict of top-level key/value objects to overwrite.
@return: The configuration object
@rtype: Configuration
"""
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
toplevel_config = {'api_key': 'fake'}
if extra_toplevel_config:
toplevel_config.update(extra_toplevel_config)
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(**toplevel_config)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2', '/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths, None)
config.parse()
# we need to delete the config dir when done
atexit.register(shutil.rmtree, config_dir)
return config
def __create_test_instance(self, configuration_logs_entry, monitors_log_configs):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, "agentConfig.json")
config_fragments_dir = os.path.join(config_dir, "configs.d")
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
fp = open(config_file, "w")
fp.write(json_lib.serialize(JsonObject(api_key="fake", logs=logs_json_array)))
fp.close()
default_paths = DefaultPaths(
"/var/log/scalyr-agent-2", "/etc/scalyr-agent-2/agent.json", "/var/lib/scalyr-agent-2"
)
config = Configuration(config_file, default_paths)
config.parse()
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def __write_checkpoint_state(self):
"""Writes the current checkpoint state to disk.
This must be done periodically to ensure that if the agent process stops and starts up again, we pick up
from where we left off copying each file.
"""
# Create the format that is expected. An overall JsonObject with the time when the file was written,
# and then an entry for each file path.
checkpoints = {}
state = {
'time': time.time(),
'checkpoints': checkpoints,
}
for processor in self.__log_processors:
checkpoints[processor.log_path] = processor.get_checkpoint()
# We write to a temporary file and then rename it to the real file name to make the write more atomic.
# We have had problems in the past with corrupted checkpoint files due to failures during the write.
file_path = os.path.join(self.__config.agent_data_path, 'checkpoints.json')
tmp_path = os.path.join(self.__config.agent_data_path, 'checkpoints.json~')
fp = None
try:
fp = open(tmp_path, 'w')
fp.write(json_lib.serialize(state))
fp.close()
fp = None
os.rename(tmp_path, file_path)
except (IOError, OSError):
if fp is not None:
fp.close()
log.exception('Could not write checkpoint file due to error', error_code='failedCheckpointWrite')
def create_configuration(extra_toplevel_config=None):
"""Creates a blank configuration file with default values. Optionally overwrites top-level key/values.
Sets api_key to 'fake' unless defined in extra_toplevel_config
@param extra_toplevel_config: Dict of top-level key/value objects to overwrite.
@return: The configuration object
@rtype: Configuration
"""
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
toplevel_config = {'api_key': 'fake'}
if extra_toplevel_config:
toplevel_config.update(extra_toplevel_config)
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(**toplevel_config)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths, None)
config.parse()
# we need to delete the config dir when done
atexit.register(shutil.rmtree, config_dir)
return config
def __create_test_instance(self, configuration_logs_entry,
monitors_log_configs):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
for entry in configuration_logs_entry:
logs_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(
json_lib.serialize(JsonObject(api_key='fake',
logs=logs_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2',
'/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
self.__monitor_fake_instances = []
for monitor_log_config in monitors_log_configs:
self.__monitor_fake_instances.append(
FakeMonitor(monitor_log_config))
# noinspection PyTypeChecker
return CopyingManager(config, self.__monitor_fake_instances)
def test_string(self):
self.__run_string_test_case('Hi there', '"Hi there"')
self.__run_string_test_case('Hi there\n', '"Hi there\\n"')
self.__run_string_test_case('Hi there\b', '"Hi there\\b"')
self.__run_string_test_case('Hi there\f', '"Hi there\\f"')
self.__run_string_test_case('Hi there\r', '"Hi there\\r"')
self.__run_string_test_case('Hi there\t', '"Hi there\\t"')
self.__run_string_test_case('Hi there\"', '"Hi there\\""')
self.__run_string_test_case('Hi there\\', '"Hi there\\\\"')
self.__run_string_test_case('Escaped\5', '"Escaped\\u0005"')
self.__run_string_test_case('Escaped\17', '"Escaped\\u000f"')
self.__run_string_test_case('Escaped\177', '"Escaped\\u007f"')
self.__run_string_test_case('Escaped\177', '"Escaped\\u007f"')
self.__run_string_test_case(u'\u2192', '"\\u2192"')
self.assertEquals(serialize('Escaped\xE2\x82\xAC', use_fast_encoding=True), '"Escaped\xe2\\u0082\xac"')
self.assertEquals(serialize('Escaped\xE2\x82\xAC', use_fast_encoding=False), '"Escaped\\u20ac"')
def test_str_conversion(self):
self.assertEquals(self.get(1, convert_to=str), "1")
self.assertEquals(self.get("ah", convert_to=str), "ah")
self.assertEquals(self.get(False, convert_to=str), "False")
self.assertEquals(self.get(1.3, convert_to=str), "1.3")
self.assertEquals(self.get(1L, convert_to=str), "1")
test_array = ["a", "b", "c"]
# str -> ArrayOfStrings (must support different variations)
arr = ArrayOfStrings(test_array)
self.assertEquals(self.get("a,b,c", convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get("a,b, c", convert_to=ArrayOfStrings), arr)
self.assertEquals(self.get('"a", "b", "c"', convert_to=ArrayOfStrings),
arr)
self.assertEquals(self.get("'a', 'b', 'c'", convert_to=ArrayOfStrings),
arr)
self.assertEquals(self.get("[a, b, c]", convert_to=ArrayOfStrings),
arr)
self.assertEquals(
self.get("['a', \"b\", c]", convert_to=ArrayOfStrings), arr)
# str -> JsonArray
self.assertEquals(
self.get(serialize(test_array), convert_to=JsonArray),
JsonArray(*test_array),
)
self.assertRaises(
BadMonitorConfiguration,
# single quotes are invalid JSON
lambda: self.assertEquals(
self.get(str(test_array), convert_to=JsonArray),
JsonArray(*test_array)),
)
# str -> JsonObject
test_obj = {"a": 1, "b": "two", "c": [1, 2, 3]}
self.assertEquals(
self.get(serialize(test_obj), convert_to=JsonObject),
json_lib_parse(serialize(test_obj)),
)
def add_thread_entry(self,
thread_id,
thread_name,
fail_if_buffer_exceeds=None):
"""Adds in a new thread entry that will be included in the post fix.
@param thread_id: The id of the thread.
@param thread_name: The name of the thread.
@param fail_if_buffer_exceeds: The maximum number of bytes that can be used by the post fix when serialized.
If this is not None, and the size will exceed this amount when the thread entry is added, then the
thread is not added and False is returned.
@type thread_id: str
@type thread_name: str
@type fail_if_buffer_exceeds: None|int
@return: True if the thread was added (can only return False if fail_if_buffer_exceeds is not None)
@rtype: bool
"""
# Calculate the size difference. It is at least the size of taken by the serialized strings.
size_difference = len(json_lib.serialize(thread_name)) + len(
json_lib.serialize(thread_id))
# Use the __per_thread_extra_bytes to calculate the additional bytes that will be consumed by serializing
# the JSON object containing the thread id and name. The number of extra bytes depends on whether or not
# there is already an entry in the JSON array, so take that into consideration.
num_threads = len(self.__threads)
if num_threads < 1:
size_difference += PostFixBuffer.__per_thread_extra_bytes[0]
else:
size_difference += PostFixBuffer.__per_thread_extra_bytes[1]
if fail_if_buffer_exceeds is not None and self.__current_size + size_difference > fail_if_buffer_exceeds:
return False
self.__current_size += size_difference
self.__threads.append({'id': thread_id, 'name': thread_name})
return True
def set_server_host(config, new_server_host):
"""Creates a new configuration file in the ``agent.d`` directory to set the ``serverHost`` server attribute
to the specified value.
@param config: The Configuration object.
@param new_server_host: The value for the ``serverHost`` server attribute.
"""
host_path = os.path.join(config.config_directory, 'server_host.json')
tmp_host_path = '%s.tmp' % host_path
try:
try:
if os.path.isfile(tmp_host_path):
os.unlink(tmp_host_path)
config_json = {
'server_attributes': {
'serverHost': new_server_host
}
}
config_content = json_lib.serialize(config_json)
tmp_file = open(tmp_host_path, 'w')
print >> tmp_file, '// Sets the server host attribute.'
print >> tmp_file, config_content
tmp_file.close()
if 'win32' == sys.platform and os.path.isfile(host_path):
os.unlink(host_path)
os.rename(tmp_host_path, host_path)
except IOError, error:
if error.errno == 13:
print >> sys.stderr, 'You do not have permission to write to the file and directory required '
print >> sys.stderr, 'to set the server host. Ensure you can write to the file at path'
print >> sys.stderr, '\'%s\' and create files in its parent directory.' % host_path
else:
print >> sys.stderr, 'Error attempting to update the server host: %s' % str(
error)
print >> sys.stderr, traceback.format_exc()
sys.exit(1)
except Exception, err:
print >> sys.stderr, 'Error attempting to update the server host attribute: %s' % str(
err)
print >> sys.stderr, traceback.format_exc()
sys.exit(1)
def set_server_host(config, new_server_host):
"""Creates a new configuration file in the ``agent.d`` directory to set the ``serverHost`` server attribute
to the specified value.
@param config: The Configuration object.
@param new_server_host: The value for the ``serverHost`` server attribute.
"""
host_path = os.path.join(config.config_directory, 'server_host.json')
tmp_host_path = '%s.tmp' % host_path
try:
try:
if os.path.isfile(tmp_host_path):
os.unlink(tmp_host_path)
config_json = {
'server_attributes': {
'serverHost': new_server_host
}
}
config_content = json_lib.serialize(config_json)
tmp_file = open(tmp_host_path, 'w')
print >>tmp_file, '// Sets the server host attribute.'
print >>tmp_file, config_content
tmp_file.close()
if 'win32' == sys.platform and os.path.isfile(host_path):
os.unlink(host_path)
os.rename(tmp_host_path, host_path)
except IOError, error:
if error.errno == 13:
print >>sys.stderr, 'You do not have permission to write to the file and directory required '
print >>sys.stderr, 'to set the server host. Ensure you can write to the file at path'
print >>sys.stderr, '\'%s\' and create files in its parent directory.' % host_path
else:
print >>sys.stderr, 'Error attempting to update the server host: %s' % str(error)
print >>sys.stderr, traceback.format_exc()
sys.exit(1)
except Exception, err:
print >>sys.stderr, 'Error attempting to update the server host attribute: %s' % str(err)
print >> sys.stderr, traceback.format_exc()
sys.exit(1)
def __create_test_instance(self, use_pipelining=False):
tmp_dir = tempfile.mkdtemp()
config_dir = os.path.join(tmp_dir, "config")
data_dir = os.path.join(tmp_dir, "data")
log_dir = os.path.join(tmp_dir, "log")
os.mkdir(data_dir)
os.mkdir(config_dir)
os.mkdir(log_dir)
self.__test_log_file = os.path.join(tmp_dir, "test.log")
fp = open(self.__test_log_file, "w")
fp.close()
config_file = os.path.join(config_dir, "agentConfig.json")
config_fragments_dir = os.path.join(config_dir, "configs.d")
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
logs_json_array.add(JsonObject(path=self.__test_log_file))
pipeline_threshold = 1.1
if use_pipelining:
pipeline_threshold = 0.0
fp = open(config_file, "w")
fp.write(
json_lib.serialize(
JsonObject(
api_key="fake",
logs=logs_json_array,
pipeline_threshold=pipeline_threshold,
)))
fp.close()
default_paths = DefaultPaths(log_dir, config_file, data_dir)
config = Configuration(config_file, default_paths, None)
config.parse()
# noinspection PyTypeChecker
self._controller = TestableCopyingManager(config, []).controller
return self._controller
def atomic_write_dict_as_json_file( file_path, tmp_path, info ):
"""Write a dict to a JSON encoded file
The file is first completely written to tmp_path, and then renamed to file_path
@param: file_path: The final path of the file
@param: tmp_path: A temporary path to write the file to
@param: info: A dict containing the JSON object to write
"""
fp = None
try:
fp = open(tmp_path, 'w')
fp.write(json_lib.serialize(info))
fp.close()
fp = None
if sys.platform == 'win32' and os.path.isfile(file_path):
os.unlink(file_path)
os.rename(tmp_path, file_path)
except (IOError, OSError):
if fp is not None:
fp.close()
log.exception('Could not write checkpoint file due to error', error_code='failedCheckpointWrite')
def content(self, cache_size=True):
"""Serialize all the information for the post fix and return it.
@param cache_size: Used for testing purposes. Can be used to turn off a slop factor that will automatically
fix differences between the calculated size and the actual size. We turn this off for testing to make
sure we catch these errors.
@type cache_size: bool
@return: The post fix to include at the end of the AddEventsRequest.
@rtype: str
"""
result = self.__format.replace('TIMESTAMP',
str(self.__client_timestamp))
result = result.replace('THREADS', json_lib.serialize(self.__threads))
# As an extra extra precaution, we update the current_size to be what it actually turned out to be. We could
# assert here to make sure it's always equal (it should be) but we don't want errors to cause issues for
# customers. Due to the way AddRequest uses this abstraction, we really really need to make sure
# the length() returns the correct result after content() was invoked, so we add in this measure to be safe.
if cache_size:
self.__current_size = len(result)
return result
def __create_test_instance(self, config_monitors, platform_monitors):
config_dir = tempfile.mkdtemp()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
monitors_json_array = JsonArray()
for entry in config_monitors:
monitors_json_array.add(JsonObject(content=entry))
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(api_key='fake', monitors=monitors_json_array)))
fp.close()
default_paths = DefaultPaths('/var/log/scalyr-agent-2', '/etc/scalyr-agent-2/agent.json',
'/var/lib/scalyr-agent-2')
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
return MonitorsManager(config, FakePlatform(platform_monitors))
def __create_test_instance(self, use_pipelining=False):
tmp_dir = tempfile.mkdtemp()
config_dir = os.path.join(tmp_dir, 'config')
data_dir = os.path.join(tmp_dir, 'data')
log_dir = os.path.join(tmp_dir, 'log')
os.mkdir(data_dir)
os.mkdir(config_dir)
os.mkdir(log_dir)
self.__test_log_file = os.path.join(tmp_dir, 'test.log')
fp = open(self.__test_log_file, 'w')
fp.close()
config_file = os.path.join(config_dir, 'agentConfig.json')
config_fragments_dir = os.path.join(config_dir, 'configs.d')
os.makedirs(config_fragments_dir)
logs_json_array = JsonArray()
logs_json_array.add(JsonObject(path=self.__test_log_file))
pipeline_threshold = 1.1
if use_pipelining:
pipeline_threshold = 0.0
fp = open(config_file, 'w')
fp.write(json_lib.serialize(JsonObject(api_key='fake', logs=logs_json_array,
pipeline_threshold=pipeline_threshold)))
fp.close()
default_paths = DefaultPaths(log_dir, config_file, data_dir)
config = Configuration(config_file, default_paths)
config.parse()
# noinspection PyTypeChecker
self._controller = TestableCopyingManager(config, []).controller
return self._controller
def __write_checkpoint_state(self):
"""Writes the current checkpoint state to disk.
This must be done periodically to ensure that if the agent process stops and starts up again, we pick up
from where we left off copying each file.
"""
# Create the format that is expected. An overall JsonObject with the time when the file was written,
# and then an entry for each file path.
checkpoints = {}
state = {
'time': time.time(),
'checkpoints': checkpoints,
}
for processor in self.__log_processors:
checkpoints[processor.log_path] = processor.get_checkpoint()
# We write to a temporary file and then rename it to the real file name to make the write more atomic.
# We have had problems in the past with corrupted checkpoint files due to failures during the write.
file_path = os.path.join(self.__config.agent_data_path,
'checkpoints.json')
tmp_path = os.path.join(self.__config.agent_data_path,
'checkpoints.json~')
fp = None
try:
fp = open(tmp_path, 'w')
fp.write(json_lib.serialize(state))
fp.close()
fp = None
if sys.platform == 'win32' and os.path.isfile(file_path):
os.unlink(file_path)
os.rename(tmp_path, file_path)
except (IOError, OSError):
if fp is not None:
fp.close()
log.exception('Could not write checkpoint file due to error',
error_code='failedCheckpointWrite')
def _fallback_json_encode( obj ):
return json_lib.serialize( obj )
def _fallback_json_encode(obj):
return json_lib.serialize(obj)
def __init__(self, base_body, max_size=1 * 1024 * 1024):
"""Initializes the instance.
@param base_body: A JsonObject or dict containing the information to send as the body of the add_events
request, with the exception of the events field. The events and client_timestamp fields must not be
included because they will be added later. Note, base_body must have some fields set, such as 'ts' which is
required by the server.
@param max_size: The maximum number of bytes this request can consume when it is serialized to JSON.
"""
assert len(base_body
) > 0, "The base_body object must have some fields defined."
assert not 'events' in base_body, "The base_body object cannot already have 'events' set."
assert not 'client_time' in base_body, "The base_body object cannot already have 'client_time' set."
# As an optimization, we use a StringIO object to serialize the request. We also
# do a little bit of the JSON object assembly by hand. Specifically, we serialize the request
# to JSON without the 'events' field, but then delete the last '}' so that we can manually
# add in the 'events: [ ... ]' ourselves. This way we can watch the size of the buffer as
# we build up events.
string_buffer = StringIO()
json_lib.serialize(base_body,
output=string_buffer,
use_fast_encoding=True)
# Now go back and find the last '}' and delete it so that we can open up the JSON again.
location = string_buffer.tell()
while location > 0:
location -= 1
string_buffer.seek(location)
if string_buffer.read(1) == '}':
break
# Now look for the first non-white character. We need to add in a comma after it.
last_char = None
while location > 0:
location -= 1
string_buffer.seek(location)
last_char = string_buffer.read(1)
if not last_char.isspace():
break
# If the character happened to a comma, back up over that since we want to write our own comma.
if location > 0 and last_char == ',':
location -= 1
if location < 0:
raise Exception(
'Could not locate trailing "}" and non-whitespace in base JSON for add events request'
)
# Now chop off everything after the character at the location.
location += 1
string_buffer.seek(location)
string_buffer.truncate()
# Append the start of our events field.
string_buffer.write(', events: [')
# This buffer keeps track of all of the stuff that must be appended after the events JSON array to terminate
# the request. That includes both the threads JSON array and the client timestamp.
self.__post_fix_buffer = PostFixBuffer(
'], threads: THREADS, client_time: TIMESTAMP }')
# The time that will be sent as the 'client_time' parameter for the addEvents request.
# This may be later updated using the set_client_time method in the case where the same AddEventsRequest
# is being reused to send the events again.
self.__post_fix_buffer.set_client_timestamp(time.time())
self.__buffer = string_buffer
self.__max_size = max_size
self.__events_added = 0
# If we have finished serializing the body, it is stored here until the close() method is invoked.
self.__body = None
def write(self, value, sort_keys=False):
return serialize(value, use_fast_encoding=True, sort_keys=sort_keys)
def __run_string_test_case(self, input_string, expected_result):
self.assertEquals(serialize(input_string, use_fast_encoding=True),
expected_result)
self.assertEquals(serialize(input_string, use_fast_encoding=False),
expected_result)
def write(self, value):
return serialize(value, use_fast_encoding=True)
def test_length_prefixed_strings_with_unicode(self):
self.assertEquals(
'`s\x00\x00\x00\x10Howdy \xe8\x92\xb8 folks!',
serialize(u'Howdy \u84b8 folks!', use_length_prefix_string=True))
def test_length_prefixed_strings(self):
self.assertEquals('`s\x00\x00\x00\x0cHowdy folks!', serialize('Howdy folks!', use_length_prefix_string=True))
def test_4byte_utf8(self):
actual = '\xF0\xAA\x9A\xA5'
expected_fast = '"\xf0\xaa\\u009a\xa5"'
expected_slow = '"\\u2a6a5"'
self.assertEquals(serialize( actual, use_fast_encoding=True), expected_fast )
self.assertEquals(serialize( actual, use_fast_encoding=False), expected_slow )
def __run_string_test_case(self, input_string, expected_result):
self.assertEquals(serialize(input_string, use_fast_encoding=True), expected_result)
self.assertEquals(serialize(input_string, use_fast_encoding=False), expected_result)
def write(self, value):
return serialize(value, use_fast_encoding=True)
def test_length_prefixed_strings_with_unicode(self):
self.assertEquals('`s\x00\x00\x00\x10Howdy \xe8\x92\xb8 folks!', serialize(u'Howdy \u84b8 folks!',
use_length_prefix_string=True))
def test_length_prefixed_strings(self):
self.assertEquals(
'`s\x00\x00\x00\x0cHowdy folks!',
serialize('Howdy folks!', use_length_prefix_string=True))
def __init__(self, base_body, max_size=1*1024*1024):
"""Initializes the instance.
@param base_body: A JsonObject or dict containing the information to send as the body of the add_events
request, with the exception of the events field. The events and client_timestamp fields must not be
included because they will be added later. Note, base_body must have some fields set, such as 'ts' which is
required by the server.
@param max_size: The maximum number of bytes this request can consume when it is serialized to JSON.
"""
assert len(base_body) > 0, "The base_body object must have some fields defined."
assert not 'events' in base_body, "The base_body object cannot already have 'events' set."
assert not 'client_time' in base_body, "The base_body object cannot already have 'client_time' set."
# As an optimization, we use a StringIO object to serialize the request. We also
# do a little bit of the JSON object assembly by hand. Specifically, we serialize the request
# to JSON without the 'events' field, but then delete the last '}' so that we can manually
# add in the 'events: [ ... ]' ourselves. This way we can watch the size of the buffer as
# we build up events.
string_buffer = StringIO()
json_lib.serialize(base_body, output=string_buffer, use_fast_encoding=True)
# Now go back and find the last '}' and delete it so that we can open up the JSON again.
location = string_buffer.tell()
while location > 0:
location -= 1
string_buffer.seek(location)
if string_buffer.read(1) == '}':
break
# Now look for the first non-white character. We need to add in a comma after it.
last_char = None
while location > 0:
location -= 1
string_buffer.seek(location)
last_char = string_buffer.read(1)
if not last_char.isspace():
break
# If the character happened to a comma, back up over that since we want to write our own comma.
if location > 0 and last_char == ',':
location -= 1
if location < 0:
raise Exception('Could not locate trailing "}" and non-whitespace in base JSON for add events request')
# Now chop off everything after the character at the location.
location += 1
string_buffer.seek(location)
string_buffer.truncate()
# Append the start of our events field.
string_buffer.write(', events: [')
# The string that must be append after all of the events to terminate the JSON. We will
# later replace TIMESTAMP with the real timestamp.
self.__post_fix = '], client_time: TIMESTAMP }'
# The time that will be sent as the 'client_time' parameter for the addEvents request.
# This may be later updated using the set_client_time method in the case where the same AddEventsRequest
# is being reused to send the events again.
self.__client_time = time.time()
self.__buffer = string_buffer
self.__max_size = max_size
self.__current_size = self.__buffer.tell() + len(self.__get_post_fix(self.__client_time))
self.__events_added = 0
# If we have finished serializing the body, it is stored here until the close() method is invoked.
self.__body = None
