def _send_payload(self):
"""
Sends current payload to backend
"""
context.log.debug(
'modified payload; current payload stats: '
'meta - %s, metrics - %s, events - %s, configs - %s' %
(len(self.payload['meta']), len(self.payload['metrics']),
len(self.payload['events']), len(self.payload['configs'])))
# Send payload to backend.
try:
self.last_http_attempt = time.time()
self._pre_process_payload() # Convert deques to lists for encoding
context.http_client.post('update/', data=self.payload)
context.default_log.debug(self.payload)
self._reset_payload() # Clear payload after successful
if self.first_run:
self.first_run = False # Set first_run to False after first successful send
if self.http_delay:
self.http_fail_count = 0
self.http_delay = 0 # Reset HTTP delay on success
context.log.debug('successful update, reset http delay')
except Exception as e:
self._post_process_payload(
) # Convert lists to deques since send failed
if isinstance(e, HTTPError) and e.response.status_code == 503:
backpressure_error = HTTP503Error(e)
context.backpressure_time = int(time.time() +
backpressure_error.delay)
context.log.debug(
'back pressure delay %s added (next talk: %s)' %
(backpressure_error.delay, context.backpressure_time))
else:
self.http_fail_count += 1
self.http_delay = exponential_delay(self.http_fail_count)
context.log.debug('http delay set to %s (fails: %s)' %
(self.http_delay, self.http_fail_count))
exception_name = e.__class__.__name__
context.log.error('failed to push data due to %s' % exception_name)
context.log.debug('additional info:', exc_info=True)
context.log.debug(
'finished flush_all; new payload stats: '
'meta - %s, metrics - %s, events - %s, configs - %s' %
(len(self.payload['meta']), len(self.payload['metrics']),
len(self.payload['events']), len(self.payload['configs'])))
def talk_to_cloud(self, root_object=None, force=False, initial=False):
"""
Asks cloud for config, object configs, filters, etc
Applies gathered data to objects and agent config
:param root_object: {} definition dict of a top object
:param force: bool will skip time check
:param initial: bool first run
"""
now = int(time.time())
if not force and (now <=
(self.last_cloud_talk_time +
context.app_config['cloud']['talk_interval'] +
self.cloud_talk_delay)
or now < context.backpressure_time):
return
# Handle root_object before explicitly initializing a root object
if not root_object:
root_object = get_root_definition()
# talk to cloud
try:
# reset the cloud talk counter to avoid sending new requests every 5.0 seconds
self.last_cloud_talk_time = int(time.time())
cloud_response = CloudResponse(
context.http_client.post('agent/', data=root_object))
if self.cloud_talk_delay:
self.cloud_talk_fails = 0
self.cloud_talk_delay = 0
context.log.debug(
'successful cloud connect, reset cloud talk delay')
except Exception as e:
if isinstance(e, HTTPError) and e.response.status_code == 503:
backpressure_error = HTTP503Error(e)
context.backpressure_time = int(time.time() +
backpressure_error.delay)
context.log.debug(
'back pressure delay %s added (next talk: %s)' %
(backpressure_error.delay, context.backpressure_time))
else:
self.cloud_talk_fails += 1
self.cloud_talk_delay = exponential_delay(
self.cloud_talk_fails)
context.log.debug(
'cloud talk delay set to %s (fails: %s)' %
(self.cloud_talk_delay, self.cloud_talk_fails))
context.log.error('could not connect to cloud', exc_info=True)
raise AmplifyCriticalException()
# check agent version status
if context.version_semver <= cloud_response.versions.obsolete:
context.log.error(
'agent is obsolete - cloud will refuse updates until it is updated (version: %s, current: %s)'
% (tuple_to_version(context.version_semver),
tuple_to_version(cloud_response.versions.current)))
self.stop()
elif context.version_semver <= cloud_response.versions.old:
context.log.warn(
'agent is old - update is recommended (version: %s, current: %s)'
% (tuple_to_version(context.version_semver),
tuple_to_version(cloud_response.versions.current)))
# set capabilities
for name, status in cloud_response.capabilities.iteritems():
name = ''.join([char.lower() for char in name if char.isalpha()])
context.capabilities[name] = status
# update special object configs and filters
changed_object_managers = set()
matched_object_configs = set()
for obj in cloud_response.objects:
object_manager = self.object_managers.get(obj.type)
if object_manager is None:
continue
if obj.id in object_manager.object_configs:
matched_object_configs.add(obj.id)
if object_manager.object_configs.get(obj.id, {}) != obj.config:
context.log.info(
'object config has changed. now "%s" %s is running with: %s'
% (obj.type, obj.id, pprint.pformat(obj.config)))
object_manager.object_configs[obj.id] = obj.config
changed_object_managers.add(obj.type)
matched_object_configs.add(obj.id)
# purge obsoleted object configs
for object_type, object_manager in self.object_managers.iteritems():
for obj_id in object_manager.object_configs.keys():
if obj_id not in matched_object_configs:
context.log.debug(
'object config has changed. now "%s" %s is running with default settings'
% (object_type, obj_id))
del object_manager.object_configs[obj_id]
changed_object_managers.add(object_type)
# don't change api_url if a custom url was set by the user in the agent config
if context.freeze_api_url:
cloud_response.config.get('cloud', {}).pop('api_url', None)
# global config changes
def _recursive_dict_match_only_existing(kwargs1, kwargs2):
for k, v1 in kwargs1.iteritems():
if isinstance(v1, dict):
v2 = kwargs2.get(k, {})
if not isinstance(v2, dict):
return False
if not _recursive_dict_match_only_existing(
v1, kwargs2.get(k, {})):
return False
else:
if v1 != kwargs2.get(str(k)):
return False
return True
config_changed = not _recursive_dict_match_only_existing(
cloud_response.config, context.app_config.default)
# apply new config
context.app_config.apply(cloud_response.config, target=0)
# perform restarts
if config_changed or len(changed_object_managers) > 0:
context.cloud_restart = True
if self.bridge_object:
self.bridge_object.flush_metrics()
if config_changed:
context.log.debug(
'app config has changed. now running with: %s' %
pprint.pformat(context.app_config.config))
context.http_client.update_cloud_url()
if self.object_managers:
for object_manager_name in reversed(
self.object_manager_order):
object_manager = self.object_managers[
object_manager_name]
object_manager.stop()
for object_manager_name in self.external_object_manager_types:
object_manager = self.object_managers[
object_manager_name]
object_manager.stop()
for name in self.external_managers.keys():
attr_string = '%s_manager' % name
thread = getattr(self, attr_string, None)
if thread is not None:
thread.kill()
elif len(changed_object_managers) > 0:
context.log.debug('obj configs changed. changed managers: %s' %
list(changed_object_managers))
for obj_type in changed_object_managers:
self.object_managers[obj_type].stop()
if not initial:
self.init_object_managers()
self.load_ext_managers()
self.last_cloud_talk_restart = int(time.time())
context.cloud_restart = False