def __get_running_containers( self, socket_file ):
"""Gets a dict of running containers that maps container id to container name
"""
self._logger.info( "Getting running containers" )
request = DockerRequest( socket_file ).get( "/containers/json" )
self._logger.info( "Get running containers response code '%d'" % request.response_code)
result = {}
if request.response_code == 200:
json = json_lib.parse( request.response_body() )
for container in json:
cid = container['Id']
if not cid == self.container_id:
try:
self._logger.info( "Get /container/%s/json" % cid )
containerRequest = DockerRequest( socket_file ).get( "/containers/%s/json" % cid )
self._logger.info( "Get containerRequest response code '%d'" % containerRequest.response_code)
if containerRequest.response_code == 200:
body = containerRequest.response_body()
containerJson = json_lib.parse( body )
result[cid] = containerJson['Name'].lstrip( '/' )
except:
result[cid] = cid
return result
def test_all_fields(self):
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message("my_message")
x.set_sampling_rate(0.5)
x.set_sequence_id(1)
x.set_sequence_number(2)
x.set_sequence_number_delta(3)
x.set_timestamp(42L)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message,sample_rate:0.5},ts:"42",si:"1",sn:2,sd:3}',
output_buffer.getvalue(),
)
self.assertEquals(
json_lib.JsonObject(
thread="foo",
ts="42",
si="1",
sn=2,
sd=3,
attrs=json_lib.JsonObject(parser="bar", message="my_message", sample_rate=0.5),
),
json_lib.parse(output_buffer.getvalue()),
)
def query_api(self, path, pretty=0):
""" Queries the k8s API at 'path', and converts OK responses to JSON objects
"""
self._ensure_session()
pretty = 'pretty=%d' % pretty
if "?" in path:
pretty = '&%s' % pretty
else:
pretty = '?%s' % pretty
url = self._http_host + path + pretty
response = self._session.get(url,
verify=self._verify_connection(),
timeout=self._timeout)
if response.status_code != 200:
global_log.log(
scalyr_logging.DEBUG_LEVEL_3,
"Invalid response from K8S API.\n\turl: %s\n\tstatus: %d\n\tresponse length: %d"
% (url, response.status_code, len(response.text)),
limit_once_per_x_secs=300,
limit_key='k8s_api_query')
raise K8sApiException(
"Invalid response from Kubernetes API when querying '%s': %s" %
(path, str(response)))
return json_lib.parse(response.text)
def run_standalone_monitor(monitor_module, monitor_python_path, monitor_config,
monitor_sample_interval, monitor_debug_level,
global_config_path):
"""Runs a single plugin monitor instance.
@param monitor_module: The name of the python module implementing the monitor.
@param monitor_python_path: The python path to search to find the module.
@param monitor_config: The monitor configuration object.
@param monitor_sample_interval: The default to use for the sample interval.
@param monitor_debug_level: The debug level to use for logging.
@param global_config_path: The path to the agent.json global configuration file to use, or None if none was
supplied.
"""
scalyr_logging.set_log_destination(use_stdout=True)
scalyr_logging.set_log_level(monitor_debug_level)
log.log(scalyr_logging.DEBUG_LEVEL_1, 'Attempting to run module %s',
monitor_module)
try:
parsed_config = json_lib.parse(monitor_config)
log.log(scalyr_logging.DEBUG_LEVEL_1,
'Parsed configuration successfully')
except json_lib.JsonParseException, e:
print >> sys.stderr, 'Failed to parse the monitor configuration as valid JSON: %s', str(
e)
return 1
def read_file_as_json(file_path):
"""Reads the entire file as a JSON value and return it.
@param file_path: the path to the file to read
@type file_path: str
@return: The JSON value contained in the file. This is typically a JsonObject, but could be primitive
values such as int or str if that is all the file contains.
@raise JsonReadFileException: If there is an error reading the file.
"""
f = None
try:
try:
if not os.path.isfile(file_path):
raise JsonReadFileException(file_path, 'The file does not exist.')
if not os.access(file_path, os.R_OK):
raise JsonReadFileException(file_path, 'The file is not readable.')
f = open(file_path, 'r')
data = f.read()
return parse(data)
except IOError, e:
raise JsonReadFileException(file_path, 'Read error occurred: ' + str(e))
except JsonParseException, e:
raise JsonReadFileException(file_path, "JSON parsing error occurred: %s (line %i, byte position %i)" % (
e.raw_message, e.line_number, e.position))
def test_all_fields(self):
x = Event(thread_id="foo", attrs={"parser": "bar"})
x.set_message("my_message")
x.set_sampling_rate(0.5)
x.set_sequence_id(1)
x.set_sequence_number(2)
x.set_sequence_number_delta(3)
x.set_timestamp(42L)
output_buffer = StringIO()
x.serialize(output_buffer)
self.assertEquals(
'{thread:"foo", attrs:{"parser":"bar",message:`s\x00\x00\x00\nmy_message,sample_rate:0.5},ts:"42",si:"1",sn:2,sd:3}',
output_buffer.getvalue(),
)
self.assertEquals(
json_lib.JsonObject(
thread="foo",
ts="42",
si="1",
sn=2,
sd=3,
attrs=json_lib.JsonObject(parser="bar",
message="my_message",
sample_rate=0.5),
),
json_lib.parse(output_buffer.getvalue()),
)
def read_file_as_json(file_path):
"""Reads the entire file as a JSON value and return it.
@param file_path: the path to the file to read
@type file_path: str
@return: The JSON value contained in the file. This is typically a JsonObject, but could be primitive
values such as int or str if that is all the file contains.
@raise JsonReadFileException: If there is an error reading the file.
"""
f = None
try:
try:
if not os.path.isfile(file_path):
raise JsonReadFileException(file_path, 'The file does not exist.')
if not os.access(file_path, os.R_OK):
raise JsonReadFileException(file_path, 'The file is not readable.')
f = open(file_path, 'r')
data = f.read()
return parse(data)
except IOError, e:
raise JsonReadFileException(file_path, 'Read error occurred: ' + str(e))
except JsonParseException, e:
raise JsonReadFileException(file_path, "JSON parsing error occurred: %s (line %i, byte position %i)" % (
e.raw_message, e.line_number, e.position))
def __extract_lines(self, request):
parsed_request = json_lib.parse(request.get_payload())
lines = []
if 'events' in parsed_request:
for event in parsed_request.get_json_array('events'):
if 'attrs' in event:
attrs = event.get_json_object('attrs')
if 'message' in attrs:
lines.append(attrs.get_string('message').strip())
return lines
def __extract_lines(self, request):
parsed_request = json_lib.parse(request.get_payload())
lines = []
if "events" in parsed_request:
for event in parsed_request.get_json_array("events"):
if "attrs" in event:
attrs = event.get_json_object("attrs")
if "message" in attrs:
lines.append(attrs.get_string("message").strip())
return lines
def __get_running_containers(self, socket_file):
"""Gets a dict of running containers that maps container id to container name
"""
request = DockerRequest(socket_file).get("/containers/json")
result = {}
if request.response_code == 200:
json = json_lib.parse(request.response_body())
for container in json:
cid = container['Id']
if not cid == self.container_id:
try:
containerRequest = DockerRequest(socket_file).get(
"/containers/%s/json" % cid)
if containerRequest.response_code == 200:
body = containerRequest.response_body()
containerJson = json_lib.parse(body)
result[cid] = containerJson['Name'].lstrip('/')
except:
result[cid] = cid
return result
def __extract_lines(self, request):
parsed_request = json_lib.parse(request.get_payload())
lines = []
if 'events' in parsed_request:
for event in parsed_request.get_json_array('events'):
if 'attrs' in event:
attrs = event.get_json_object('attrs')
if 'message' in attrs:
lines.append(attrs.get_string('message').strip())
return lines
def run_standalone_monitor(monitor_module, monitor_python_path, monitor_config, monitor_sample_interval):
"""Runs a single plugin monitor instance.
@param monitor_module: The name of the python module implementing the monitor.
@param monitor_python_path: The python path to search to find the module.
@param monitor_config: The monitor configuration object.
@param monitor_sample_interval: The default to use for the sample interval.
"""
scalyr_logging.set_log_destination(use_stdout=True)
try:
parsed_config = json_lib.parse(monitor_config)
except json_lib.JsonParseException, e:
print >>sys.stderr, 'Failed to parse the monitor configuration as valid JSON: %s', str(e)
return 1
def query_api(self, path):
""" Queries the kubelet API at 'path', and converts OK responses to JSON objects
"""
url = self._http_host + path
response = self._session.get(url, timeout=self._timeout)
if response.status_code != 200:
global_log.log(
scalyr_logging.DEBUG_LEVEL_3,
"Invalid response from Kubelet API.\n\turl: %s\n\tstatus: %d\n\tresponse length: %d"
% (url, response.status_code, len(response.text)),
limit_once_per_x_secs=300,
limit_key='kubelet_api_query')
raise KubeletApiException(
"Invalid response from Kubelet API when querying '%s': %s" %
(path, str(response)))
return json_lib.parse(response.text)
def __get_scalyr_container_id(self, socket_file):
"""Gets the container id of the scalyr-agent container
If the config option container_name is empty, then it is assumed that the scalyr agent is running
on the host and not in a container and None is returned.
"""
result = None
name = self._config.get('container_name')
if name:
request = DockerRequest(socket_file).get("/containers/%s/json" %
name)
if request.response_code == 200:
json = json_lib.parse(request.response_body())
result = json['Id']
if not result:
raise Exception(
"Unabled to find a matching container id for container '%s'. Please make sure that a container named '%s' is running."
% (name, name))
return result
def __get_scalyr_container_id( self, socket_file ):
"""Gets the container id of the scalyr-agent container
If the config option container_name is empty, then it is assumed that the scalyr agent is running
on the host and not in a container and None is returned.
"""
result = None
name = self._config.get( 'container_name' )
if name:
self._logger.info( "GET /containers/%s/json" % name)
request = DockerRequest( socket_file ).get( "/containers/%s/json" % name )
if request.response_code == 200:
self._logger.info( "Get a 200 response code" )
json = json_lib.parse( request.response_body() )
result = json['Id']
if not result:
self._logger.info( "Got a non-200 response code: '%d'" % request.response_code )
raise Exception( "Unable to find a matching container id for container '%s'. Please make sure that a container named '%s' is running." % (name, name) )
return result
def run_standalone_monitor(monitor_module, monitor_python_path, monitor_config, monitor_sample_interval,
monitor_debug_level, global_config_path):
"""Runs a single plugin monitor instance.
@param monitor_module: The name of the python module implementing the monitor.
@param monitor_python_path: The python path to search to find the module.
@param monitor_config: The monitor configuration object.
@param monitor_sample_interval: The default to use for the sample interval.
@param monitor_debug_level: The debug level to use for logging.
@param global_config_path: The path to the agent.json global configuration file to use, or None if none was
supplied.
"""
scalyr_logging.set_log_destination(use_stdout=True)
scalyr_logging.set_log_level(monitor_debug_level)
log.log(scalyr_logging.DEBUG_LEVEL_1, 'Attempting to run module %s', monitor_module)
try:
parsed_config = json_lib.parse(monitor_config)
log.log(scalyr_logging.DEBUG_LEVEL_1, 'Parsed configuration successfully')
except json_lib.JsonParseException, e:
print >>sys.stderr, 'Failed to parse the monitor configuration as valid JSON: %s', str(e)
return 1
class ScalyrClientSession(object):
"""Encapsulates the connection between the agent and the Scalyr servers.
It is a session in that we generally only have one connection open to the Scalyr servers at any given time.
The session aspect is important because we must ensure that the timestamps we include in the AddEventRequests
are monotonically increasing within a session.
"""
def __init__(self,
server,
api_key,
agent_version,
quiet=False,
request_deadline=60.0,
ca_file=None):
"""Initializes the connection.
This does not actually try to connect to the server.
@param server: The URL for the server to send requests to, such as https://agent.scalyr.com
@param api_key: The write logs key to use to authenticate all requests from this agent to scalyr.
It both authenticates the requests and identifies which account it belongs to.
@param agent_version: The agent version number, which is included in request headers sent to the server.
@param quiet: If True, will not log non-error information.
@param request_deadline: The maximum time to wait for all requests in seconds.
@param ca_file: The path to the file containing the certificates for the trusted certificate authority roots.
This is used for the SSL connections to verify the connection is to Scalyr.
@type server: str
@type api_key: str
@type agent_version: str
@type quiet: bool
@type request_deadline: float
@type ca_file: str
"""
if not quiet:
log.info('Using "%s" as address for scalyr servers' % server)
# Verify the server address looks right.
parsed_server = re.match('^(http://|https://|)([^:]*)(:\d+|)$',
server.lower())
if parsed_server is None:
raise Exception('Could not parse server address "%s"' % server)
# The full URL address
self.__full_address = server
# The host for the server.
self.__host = parsed_server.group(2)
# Whether or not the connection uses SSL. For production use, this should always be true. We only
# use non-SSL when testing against development versions of the Scalyr server.
self.__use_ssl = parsed_server.group(1) == 'https://'
# Determine the port, defaulting to the right one based on protocol if not given.
if parsed_server.group(3) != '':
self.__port = int(parsed_server.group(3)[1:])
elif self.__use_ssl:
self.__port = 443
else:
self.__port = 80
# The HTTPConnection object that has been opened to the servers, if one has been opened.
self.__connection = None
self.__api_key = api_key
self.__session_id = scalyr_util.create_unique_id()
# The time of the last success.
self.__last_success = None
# The version number of the installed agent
self.__agent_version = agent_version
# The last time the connection was closed, if any.
self.__last_connection_close = None
# We create a few headers ahead of time so that we don't have to recreate them each time we need them.
self.__standard_headers = {
'Connection': 'Keep-Alive',
'Accept': 'application/json',
'User-Agent': ScalyrClientSession.__get_user_agent(agent_version)
}
# The number of seconds to wait for a blocking operation on the connection before considering it to have
# timed out.
self.__request_deadline = request_deadline
# The total number of RPC requests sent.
self.total_requests_sent = 0
# The total number of RPC requests that failed.
self.total_requests_failed = 0
# The total number of bytes sent over the network.
self.total_request_bytes_sent = 0
# The total number of bytes received.
self.total_response_bytes_received = 0
# The total number of secs spent waiting for a responses (so average latency can be calculated by dividing
# this number by self.total_requests_sent). This includes connection establishment time.
self.total_request_latency_secs = 0
# The total number of HTTP connections successfully created.
self.total_connections_created = 0
# The path the file containing the certs for the root certificate authority to use for verifying the SSL
# connection to Scalyr. If this is None, then server certificate verification is disabled, and we are
# susceptible to man-in-the-middle attacks.
self.__ca_file = ca_file
def ping(self):
"""Ping the Scalyr server by sending a test message to add zero events.
If the returned message is 'success', then it has been verified that the agent can connect to the
configured Scalyr server and that the api key is correct.
@return: The status message returned by the server.
@rtype:
"""
return self.send(self.add_events_request())[0]
def __send_request(self,
request_path,
body=None,
body_func=None,
is_post=True):
"""Sends a request either using POST or GET to Scalyr at the specified request path. It may be either
a POST or GET.
Parses, returns response.
@param request_path: The path of the URL to post to.
@param [body]: The body string to send. May be None if body_func is specified. Ignored if not POST.
@param [body_func]: A function that will be invoked to retrieve the body to send in the post. Ignored if not
POST.
@param [is_post]: True if this request should be sent using a POST, otherwise GET.
@type request_path: str
@type body: str|None
@type body_func: func|None
@type is_post: bool
@return: A tuple containing the status message in the response (such as 'success'), the number of bytes
sent, and the full response.
@rtype: (str, int, str)
"""
current_time = time.time()
# Refuse to try to send the message if the connection has been recently closed and we have not waited
# long enough to try to re-open it. We do this to avoid excessive connection opens and SYN floods.
if self.__last_connection_close is not None and current_time - self.__last_connection_close < 30:
return 'client/connectionClosed', 0, ''
self.total_requests_sent += 1
was_success = False
bytes_received = 0
if self.__use_ssl:
if not __has_ssl__:
log.warn(
'No ssl library available so cannot verify server certificate when communicating with Scalyr. '
'This means traffic is encrypted but can be intercepted through a man-in-the-middle attack. '
'To solve this, install the Python ssl library. '
'For more details, see https://www.scalyr.com/help/scalyr-agent#ssl',
limit_once_per_x_secs=86400,
limit_key='nosslwarning',
error_code='client/nossl')
elif self.__ca_file is None:
log.warn(
'Server certificate validation has been disabled while communicating with Scalyr. '
'This means traffic is encrypted but can be intercepted through a man-in-the-middle attach. '
'Please update your configuration file to re-enable server certificate validation.',
limit_once_per_x_secs=86400,
limit_key='nocertwarning',
error_code='client/sslverifyoff')
response = ''
try:
try:
if self.__connection is None:
if self.__use_ssl:
# If we do not have the SSL library, then we cannot do server certificate validation anyway.
if __has_ssl__:
ca_file = self.__ca_file
else:
ca_file = None
self.__connection = HTTPSConnectionWithTimeoutAndVerification(
self.__host, self.__port, self.__request_deadline,
ca_file, __has_ssl__)
else:
self.__connection = HTTPConnectionWithTimeout(
self.__host, self.__port, self.__request_deadline)
self.__connection.connect()
self.total_connections_created += 1
except (socket.error, socket.herror, socket.gaierror), error:
if hasattr(error, 'errno'):
errno = error.errno
else:
errno = None
if __has_ssl__ and isinstance(error, ssl.SSLError):
log.error(
'Failed to connect to "%s" due to some SSL error. Possibly the configured certificate '
'for the root Certificate Authority could not be parsed, or we attempted to connect to '
'a server whose certificate could not be trusted (if so, maybe Scalyr\'s SSL cert has '
'changed and you should update your agent to get the new certificate). The returned '
'errno was %d and the full exception was \'%s\'. Closing connection, will re-attempt',
self.__full_address,
errno,
str(error),
error_code='client/connectionFailed')
elif errno == 61: # Connection refused
log.error(
'Failed to connect to "%s" because connection was refused. Server may be unavailable.',
self.__full_address,
error_code='client/connectionFailed')
elif errno == 8: # Unknown name
log.error(
'Failed to connect to "%s" because could not resolve address. Server host may be bad.',
self.__full_address,
error_code='client/connectionFailed')
elif errno is not None:
log.error(
'Failed to connect to "%s" due to errno=%d. Exception was %s. Closing connection, '
'will re-attempt',
self.__full_address,
errno,
str(error),
error_code='client/connectionFailed')
else:
log.error(
'Failed to connect to "%s" due to exception. Exception was %s. Closing connection, '
'will re-attempt',
self.__full_address,
str(error),
error_code='client/connectionFailed')
return 'client/connectionFailed', 0, ''
if is_post:
if body is None:
body_str = body_func()
else:
body_str = body
else:
body_str = ""
self.total_request_bytes_sent += len(body_str) + len(request_path)
# noinspection PyBroadException
try:
if is_post:
log.log(scalyr_logging.DEBUG_LEVEL_5,
'Sending POST %s with body \"%s\"', request_path,
body_str)
self.__connection.request('POST',
request_path,
body=body_str,
headers=self.__standard_headers)
else:
log.log(scalyr_logging.DEBUG_LEVEL_5, 'Sending GET %s',
request_path)
self.__connection.request('GET',
request_path,
headers=self.__standard_headers)
response = self.__connection.getresponse().read()
bytes_received = len(response)
except Exception, error:
# TODO: Do not just catch Exception. Do narrower scope.
if hasattr(error, 'errno'):
log.error(
'Failed to connect to "%s" due to errno=%d. Exception was %s. Closing connection, '
'will re-attempt',
self.__full_address,
error.errno,
str(error),
error_code='client/requestFailed')
else:
log.exception(
'Failed to send request due to exception. Closing connection, will re-attempt',
error_code='requestFailed')
return 'requestFailed', len(body_str), response
log.log(scalyr_logging.DEBUG_LEVEL_5,
'Response was received with body \"%s\"', response)
# If we got back an empty result, that often means the connection has been closed or reset.
if len(response) == 0:
log.error(
'Received empty response, server may have reset connection. Will re-attempt',
error_code='emptyResponse')
return 'emptyResponse', len(body_str), response
# Try to parse the response
# noinspection PyBroadException
try:
response_as_json = json_lib.parse(response)
except Exception:
# TODO: Do not just catch Exception. Do narrower scope. Also, log error here.
log.exception(
'Failed to parse response of \'%s\' due to exception. Closing connection, will '
're-attempt',
scalyr_util.remove_newlines_and_truncate(response, 1000),
error_code='parseResponseFailed')
return 'parseResponseFailed', len(body_str), response
self.__last_success = current_time
if 'status' in response_as_json:
status = response_as_json['status']
if status == 'success':
was_success = True
elif status == 'error/client/badParam':
log.error(
'Request to \'%s\' failed due to a bad parameter value. This may be caused by an '
'invalid write logs api key in the configuration',
self.__full_address,
error_code='error/client/badParam')
else:
log.error(
'Request to \'%s\' failed due to an error. Returned error code was \'%s\'',
self.__full_address,
status,
error_code='error/client/badParam')
return status, len(body_str), response
else:
log.error(
'No status message provided in response. Unknown error. Response was \'%s\'',
scalyr_util.remove_newlines_and_truncate(response, 1000),
error_code='unknownError')
return 'unknownError', len(body_str), response
""", threads: [], client_time: 100 }""")
request.set_client_time(2)
self.assertEquals(
request.get_payload(),
"""{"token":"fakeToken", events: [{"name":"eventOne","ts":"1"},{"name":"eventTwo","ts":"2"}]"""
""", threads: [], client_time: 2 }""")
request.close()
def test_sequence_id_but_no_number( self ):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(request.add_event({'name': 'eventOne'}, timestamp=1L, sequence_id='1234'))
self.assertEquals(request.total_events, 1)
json = json_lib.parse( request.get_payload() )
event = json['events'][0]
self.assertFalse( 'si' in event )
self.assertFalse( 'sn' in event )
self.assertFalse( 'sd' in event )
request.close()
def test_sequence_number_but_no_id( self ):
request = AddEventsRequest(self.__body)
request.set_client_time(1)
self.assertTrue(request.add_event({'name': 'eventOne'}, timestamp=1L, sequence_number=1234))
self.assertEquals(request.total_events, 1)
json = json_lib.parse( request.get_payload() )
event = json['events'][0]
def _fallback_json_decode( text ):
return json_lib.parse( text )
def upgrade_windows_install(config, release_track="stable", preserve_msi=False, use_ui=True):
"""Performs an upgrade for an existing Scalyr Agent 2 that was previously installed using a Windows MSI install
file.
This will contact the Scalyr servers to see what the most up-to-date version of the agent is and, if necessary,
download an MSI file.
@param config: The configuration for this agent.
@param release_track: The release track to use when checking which version is the latest.
@param preserve_msi: Whether or not to delete the MSI file once the upgrade is finished. Note, this
argument is essentially ignored for now and we always leave the file because we cannot delete it with
the current way we exec the msiexec process.
@param use_ui: Whether or not the msiexec upgrade command should be run with the UI.
@rtype config: Configuration
@rtype release_track: str
@rtype preserve_msi: bool
@rtype use_ui: bool
@return: The exit status code.
"""
# The URL path of the agent to upgrade to.
url_path = None
try:
platform_controller = PlatformController.new_platform()
my_default_paths = platform_controller.default_paths
# Ensure agent was installed via MSI
if MSI_INSTALL != platform_controller.install_type:
raise UpgradeFailure('The current agent was not installed via MSI, so you may not use the upgrade windows '
'command.')
# Ensure that the user has not changed the defaults for the config, data, and log directory.
if my_default_paths.config_file_path != config.file_path:
raise UpgradeFailure('The agent is not using the default configuration file so you may not use the '
'upgrade windows command.')
if my_default_paths.agent_data_path != config.agent_data_path:
raise UpgradeFailure('The agent is not using the default data directory so you may not use the upgrade '
'windows command.')
if my_default_paths.agent_log_path != config.agent_log_path:
raise UpgradeFailure('The agent is not using the default log directory so you may not use the upgrade '
'windows command.')
# Determine if a newer version is available
client = ScalyrClientSession(config.scalyr_server, config.api_key, SCALYR_VERSION, quiet=True,
ca_file=config.ca_cert_path)
status, size, response = client.perform_agent_version_check(release_track)
if status.lower() != 'success':
raise UpgradeFailure('Failed to contact the Scalyr servers to check for latest update. Error code '
'was "%s"' % status)
# TODO: We shouldn't have to reparse response on JSON, but for now that, that's what the client library
# does.
data_payload = json_lib.parse(response)['data']
if not data_payload['update_required']:
print 'The latest version is already installed.'
return 0
print 'Attempting to upgrade agent from version %s to version %s.' % (SCALYR_VERSION,
data_payload['current_version'])
url_path = data_payload['urls']['win32']
file_portion = url_path[url_path.rfind('/')+1:]
download_location = os.path.join(tempfile.gettempdir(), file_portion)
try:
try:
print 'Downloading agent from %s.' % url_path
urllib.urlretrieve(url_path, download_location)
if not os.path.isfile(download_location):
raise UpgradeFailure('Failed to download installation package')
if use_ui:
print ('Executing upgrade. Please follow the instructions in the subsequent dialog boxes to '
'complete the upgrade process.')
else:
print ('Executing upgrade. It will finish in the background.')
# Because this file, config_main.py, is part of the currently installed Scalyr Agent package, we have
# to finish our use of it before the upgrade can proceed. So, we just fork off the msiexec process
# in detached mode and terminate this program. This means we cannot report any errors that happen
# here, but I don't see a way around this for now.
# noinspection PyUnresolvedReferences
from win32process import DETACHED_PROCESS
upgrade_command = ['msiexec.exe', '/i', "{}".format(download_location)]
if not use_ui:
upgrade_command.append('/qn')
subprocess.Popen(upgrade_command,
shell=False, stdin=None, stdout=None, stderr=None, close_fds=True,
creationflags=DETACHED_PROCESS)
return 0
except IOError, error:
raise UpgradeFailure('Could not download the installer, returned error %s' % str(error))
finally:
# TODO: Actually delete the temporary file. We cannot right now since our execution finishes
# before the msiexec process runs, but maybe we can do something like have a small shell script
# that runs the upgrader and then deletes the file. Something to consider post-alpha release.
if preserve_msi:
print 'Downloaded installer file has been left at %s' % download_location
except UpgradeFailure, error:
print >>sys.stderr
print >>sys.stderr, 'The upgrade failed due to the following reason: %s' % error.message
if url_path is not None:
print >>sys.stderr, 'You may try downloading and running the installer file yourself.'
print >>sys.stderr, 'The installer can be downloaded from %s' % url_path
print >>sys.stderr, 'Please e-mail [email protected] for help resolving this issue.'
return 1
def convert_config_param(field_name, value, convert_to, is_environment_variable=False):
"""Convert monitor config values to a different type according to the ALLOWED_CONVERSIONS matrix"""
convert_from = type(value)
kind = 'environment variable'
if not is_environment_variable:
kind = 'config param'
conversion_allowed = False
if convert_from in ALLOWED_CONVERSIONS:
if convert_to in set([convert_from]) | ALLOWED_CONVERSIONS[convert_from]:
conversion_allowed = True
if not conversion_allowed:
raise BadConfiguration('Prohibited conversion of %s "%s" from %s to %s' %
(kind, field_name, convert_from, convert_to),
field_name, 'illegalConversion')
# If no type change, simply return unconverted value
if convert_from == convert_to:
return value
# Anything is allowed to go to str/unicode
if convert_to in STRING_TYPES:
return convert_to(value)
if convert_from == list and convert_to == JsonArray:
try:
return JsonArray(*value)
except JsonConversionException:
raise BadConfiguration(
'Could not convert value %s for field "%s" from %s to %s'
% (value, field_name, convert_from, convert_to),
field_name, 'notJsonArray')
if convert_from in (list, JsonArray) and convert_to == ArrayOfStrings:
list_of_strings = []
for item in value:
if type(item) not in STRING_TYPES:
raise BadConfiguration(
'Non-string element found in value %s for field "%s"' % (value, field_name),
field_name, 'notArrayOfStrings')
list_of_strings.append(item)
return ArrayOfStrings(*list_of_strings)
# Anything is allowed to go from string/unicode to the conversion type, as long as it can be parsed.
# Special-case handle bool and JsonArray
if convert_from in STRING_TYPES:
if convert_to == bool:
return str(value).lower() == 'true'
elif convert_to in (JsonArray, JsonObject):
try:
return json_lib.parse(value)
except JsonParseException:
raise BadConfiguration(
'Could not parse value %s for field "%s" as %s' % (value, field_name, convert_to),
field_name, 'notJsonObject')
elif convert_to == ArrayOfStrings:
try:
return parse_array_of_strings(value)
except TypeError:
raise BadConfiguration(
'Could not parse value %s for field "%s" as %s' % (value, field_name, convert_to),
field_name, 'notArrayOfStrings')
elif convert_to in NUMERIC_TYPES:
try:
return convert_to(value)
except ValueError:
raise BadConfiguration('Could not parse value %s for field "%s" as numeric type %s' % (
value, field_name, convert_to), field_name, 'notNumber')
if convert_from not in NUMERIC_TYPES:
raise BadConfiguration('Type conversion for field "%s" from %s to %s not implemented.' %
(field_name, convert_from, convert_to), field_name, 'notNumber')
if convert_to == bool:
raise BadConfiguration('A numeric value %s was given for boolean field "%s"' % (
value, field_name), field_name, 'notBoolean')
if convert_to not in NUMERIC_TYPES:
raise BadConfiguration('Type conversion for field "%s" from %s to %s not implemented.' %
(field_name, convert_from, convert_to), field_name, 'unsupportedConversion')
# At this point, we are trying to convert a number to another number type. We only allow int to long
# and long, int to float.
if convert_to == float and convert_from in (long, int):
return float(value)
if convert_to == long and convert_from == int:
return long(value)
raise BadConfiguration('A numeric value of %s was given for field "%s" but a %s is required.' % (
value, field_name, convert_to), field_name, 'wrongType')
def upgrade_windows_install(config,
release_track="stable",
preserve_msi=False,
use_ui=True):
"""Performs an upgrade for an existing Scalyr Agent 2 that was previously installed using a Windows MSI install
file.
This will contact the Scalyr servers to see what the most up-to-date version of the agent is and, if necessary,
download an MSI file.
@param config: The configuration for this agent.
@param release_track: The release track to use when checking which version is the latest.
@param preserve_msi: Whether or not to delete the MSI file once the upgrade is finished. Note, this
argument is essentially ignored for now and we always leave the file because we cannot delete it with
the current way we exec the msiexec process.
@param use_ui: Whether or not the msiexec upgrade command should be run with the UI.
@rtype config: Configuration
@rtype release_track: str
@rtype preserve_msi: bool
@rtype use_ui: bool
@return: The exit status code.
"""
# The URL path of the agent to upgrade to.
url_path = None
try:
platform_controller = PlatformController.new_platform()
my_default_paths = platform_controller.default_paths
# Ensure agent was installed via MSI
if MSI_INSTALL != platform_controller.install_type:
raise UpgradeFailure(
'The current agent was not installed via MSI, so you may not use the upgrade windows '
'command.')
# Ensure that the user has not changed the defaults for the config, data, and log directory.
if my_default_paths.config_file_path != config.file_path:
raise UpgradeFailure(
'The agent is not using the default configuration file so you may not use the '
'upgrade windows command.')
if my_default_paths.agent_data_path != config.agent_data_path:
raise UpgradeFailure(
'The agent is not using the default data directory so you may not use the upgrade '
'windows command.')
if my_default_paths.agent_log_path != config.agent_log_path:
raise UpgradeFailure(
'The agent is not using the default log directory so you may not use the upgrade '
'windows command.')
# Determine if a newer version is available
client = ScalyrClientSession(config.scalyr_server,
config.api_key,
SCALYR_VERSION,
quiet=True,
ca_file=config.ca_cert_path)
status, size, response = client.perform_agent_version_check(
release_track)
if status.lower() != 'success':
raise UpgradeFailure(
'Failed to contact the Scalyr servers to check for latest update. Error code '
'was "%s"' % status)
# TODO: We shouldn't have to reparse response on JSON, but for now that, that's what the client library
# does.
data_payload = json_lib.parse(response)['data']
if not data_payload['update_required']:
print 'The latest version is already installed.'
return 0
print 'Attempting to upgrade agent from version %s to version %s.' % (
SCALYR_VERSION, data_payload['current_version'])
url_path = data_payload['urls']['win32']
file_portion = url_path[url_path.rfind('/') + 1:]
download_location = os.path.join(tempfile.gettempdir(), file_portion)
try:
try:
print 'Downloading agent from %s.' % url_path
urllib.urlretrieve(url_path, download_location)
if not os.path.isfile(download_location):
raise UpgradeFailure(
'Failed to download installation package')
if use_ui:
print(
'Executing upgrade. Please follow the instructions in the subsequent dialog boxes to '
'complete the upgrade process.')
else:
print(
'Executing upgrade. It will finish in the background.'
)
# Because this file, config_main.py, is part of the currently installed Scalyr Agent package, we have
# to finish our use of it before the upgrade can proceed. So, we just fork off the msiexec process
# in detached mode and terminate this program. This means we cannot report any errors that happen
# here, but I don't see a way around this for now.
# noinspection PyUnresolvedReferences
from win32process import DETACHED_PROCESS
upgrade_command = [
'msiexec.exe', '/i', "{}".format(download_location)
]
if not use_ui:
upgrade_command.append('/qn')
subprocess.Popen(upgrade_command,
shell=False,
stdin=None,
stdout=None,
stderr=None,
close_fds=True,
creationflags=DETACHED_PROCESS)
return 0
except IOError, error:
raise UpgradeFailure(
'Could not download the installer, returned error %s' %
str(error))
finally:
# TODO: Actually delete the temporary file. We cannot right now since our execution finishes
# before the msiexec process runs, but maybe we can do something like have a small shell script
# that runs the upgrader and then deletes the file. Something to consider post-alpha release.
if preserve_msi:
print 'Downloaded installer file has been left at %s' % download_location
except UpgradeFailure, error:
print >> sys.stderr
print >> sys.stderr, 'The upgrade failed due to the following reason: %s' % error.message
if url_path is not None:
print >> sys.stderr, 'You may try downloading and running the installer file yourself.'
print >> sys.stderr, 'The installer can be downloaded from %s' % url_path
print >> sys.stderr, 'Please e-mail [email protected] for help resolving this issue.'
return 1
def _fallback_json_decode(text):
return json_lib.parse(text)
def convert_config_param(field_name,
value,
convert_to,
is_environment_variable=False):
"""Convert monitor config values to a different type according to the ALLOWED_CONVERSIONS matrix
None is an invalid input and will raise BadConfiguration error.
Empty strings will convert into str, bool, ArrayOfStrings, SpaceAndCommaSeparatedArrayOfStrings but raises
exception for int, float, JsonArray, JsonObject
"""
convert_from = type(value)
kind = "environment variable"
if not is_environment_variable:
kind = "config param"
conversion_allowed = False
if convert_from in ALLOWED_CONVERSIONS:
if convert_to in set([convert_from
]) | ALLOWED_CONVERSIONS[convert_from]:
conversion_allowed = True
if not conversion_allowed:
raise BadConfiguration(
'Prohibited conversion of %s "%s" from %s to %s' %
(kind, field_name, convert_from, convert_to),
field_name,
"illegalConversion",
)
# If no type change, simply return unconverted value
if convert_from == convert_to:
return value
# Anything is allowed to go to str/unicode
if convert_to in STRING_TYPES:
return convert_to(value)
if convert_from == list and convert_to == JsonArray:
try:
return JsonArray(*value)
except JsonConversionException:
raise BadConfiguration(
'Could not convert value %s for field "%s" from %s to %s' %
(value, field_name, convert_from, convert_to),
field_name,
"notJsonArray",
)
if convert_from in (list, JsonArray) and convert_to in (
ArrayOfStrings,
SpaceAndCommaSeparatedArrayOfStrings,
):
list_of_strings = []
for item in value:
if type(item) not in STRING_TYPES:
raise BadConfiguration(
'Non-string element found in value %s for field "%s"' %
(value, field_name),
field_name,
"notArrayOfStrings",
)
list_of_strings.append(item)
return convert_to(list_of_strings)
# Anything is allowed to go from string/unicode to the conversion type, as long as it can be parsed.
# Special-case handle bool and JsonArray
if convert_from in STRING_TYPES:
if convert_to == bool:
return str(value).lower() == "true"
elif convert_to in (JsonArray, JsonObject):
try:
return json_lib.parse(value)
except JsonParseException:
raise BadConfiguration(
'Could not parse value %s for field "%s" as %s' %
(value, field_name, convert_to),
field_name,
"notJsonObject",
)
elif convert_to in (ArrayOfStrings,
SpaceAndCommaSeparatedArrayOfStrings):
try:
# ArrayOfStrings and it's
return parse_array_of_strings(value, convert_to.separators)
except TypeError:
raise BadConfiguration(
'Could not parse value %s for field "%s" as %s' %
(value, field_name, convert_to),
field_name,
"notArrayOfStrings",
)
elif convert_to in NUMERIC_TYPES:
try:
return convert_to(value)
except ValueError:
raise BadConfiguration(
'Could not parse value %s for field "%s" as numeric type %s'
% (value, field_name, convert_to),
field_name,
"notNumber",
)
if convert_from not in NUMERIC_TYPES:
raise BadConfiguration(
'Type conversion for field "%s" from %s to %s not implemented.' %
(field_name, convert_from, convert_to),
field_name,
"notNumber",
)
if convert_to == bool:
raise BadConfiguration(
'A numeric value %s was given for boolean field "%s"' %
(value, field_name),
field_name,
"notBoolean",
)
if convert_to not in NUMERIC_TYPES:
raise BadConfiguration(
'Type conversion for field "%s" from %s to %s not implemented.' %
(field_name, convert_from, convert_to),
field_name,
"unsupportedConversion",
)
# At this point, we are trying to convert a number to another number type. We only allow int to long
# and long, int to float.
if convert_to == float and convert_from in (long, int):
return float(value)
if convert_to == long and convert_from == int:
return long(value)
raise BadConfiguration(
'A numeric value of %s was given for field "%s" but a %s is required.'
% (value, field_name, convert_to),
field_name,
"wrongType",
)
