def __init__(self, config=None):
"""
"""
console.info("Start python agent with version %s",
global_settings().agent_version)
self._last_harvest = 0.0
self._harvest_thread = threading.Thread(target=self._harvest_loop,
name="TingYunHarvestThread")
self._harvest_thread.setDaemon(True) # should take caution to the doc
self._harvest_shutdown = threading.Event()
self._config = config if config is not None else global_settings()
self._lock = threading.Lock()
self._process_shutdown = False
self._applications = {} # used to store the actived applications
self.__max_tracker = 5000
self.__data_sampler = []
if self._config.enabled:
atexit.register(self._atexit_shutdown)
# Register our exit event to the uwsgi shutdown hooks.
if 'uwsgi' in sys.modules:
import uwsgi
uwsgi_original_atexit_callback = getattr(uwsgi, 'atexit', None)
def uwsgi_atexit_callback():
self._atexit_shutdown()
if uwsgi_original_atexit_callback:
uwsgi_original_atexit_callback()
uwsgi.atexit = uwsgi_atexit_callback
def singleton_instance():
"""
:return: singleton instance
"""
if Dispatcher._instance:
print("sunyan: second time, Dispatcher._instance is not None.")
return Dispatcher._instance
print("sunyan: first time, Dispatcher._instance is None.")
instance = None
with Dispatcher._instance_lock:
if not Dispatcher._instance:
instance = Dispatcher(global_settings())
Dispatcher._instance = instance
console.info('Creating instance of agent dispatcher')
if instance:
# FIXME
#instance.register_data_sampler(memory_usage_sampler, ('memory.usage',))
instance.register_data_sampler(cpu_usage_sampler, ('cpu.usage', ))
instance.active_dispatcher()
return instance
def __init__(self, dispatcher, app_name, link_app_name=None):
self.enabled = True
self._app_name = app_name
self._link_app_name = link_app_name if link_app_name else []
if not app_name:
self._app_name = global_settings().app_name
if not dispatcher:
dispatcher = dispatcher_instance()
console.debug("init application with new dispatcher.")
self._dispatcher = dispatcher
def __init__(self, url, config, license_key, redirect_host):
self.url = url
self.config = config
self.license_key = license_key
self.local_setting = global_settings()
self.merged_settings = config
self.redirect_host = redirect_host
self._session = None
self.request_param = {
"licenseKey": self.license_key,
"version": self.local_setting.data_version,
"appSessionKey": self.config.appSessionKey,
}
def __init__(self, config_file):
self.config_file = config_file
self.is_embattled = False
self.valid_embattle = True
self._config_parser = ConfigParser.RawConfigParser()
self._settings = global_settings()
self._inspect_lock = threading.Lock()
if not config_file:
self.config_file = os.environ.get(ENV_CONFIG_FILE, None)
if not self.config_file:
log_bootstrap('Agent config file is not found, agent start failed.')
self.valid_embattle = False
log_bootstrap("get config file %s" % self.config_file)
def trace_node(self, root):
"""
:param root: the root node of the tracker
:return:
"""
dbtype = r" %s" % self.dbtype
params = {"sql": "", "explainPlan": {}, "stacktrace": []}
children = []
call_count = 1
class_name = ""
method_name = root.name
root.trace_node_count += 1
start_time = node_start_time(root, self)
end_time = node_end_time(root, self)
operation = str(self.operation).upper()
metric_name = "Database%s/CALL/sql" % dbtype
call_url = metric_name
if operation in ('SELECT', 'UPDATE', 'INSERT', 'DELETE'):
metric_name = 'Database/%s/%s' % (
self.table,
operation) if self.table else 'Database/NULL/%s' % operation
if self.formatted:
# Note, use local setting only.
_settings = global_settings()
params['sql'] = self.formatted
if _settings.action_tracer.log_sql:
console.info(
"Log sql is opened. sql upload is disabled, sql sentence is %s",
self.formatted)
params['sql'] = ""
if self.explain_plan:
params['explainPlan'] = self.explain_plan
if self.stack_trace:
for line in self.stack_trace:
if len(line) >= 4 and 'tingyun' not in line[0]:
params['stacktrace'].append(
"%s(%s:%s)" % (line[2], line[0], line[1]))
return [
start_time, end_time, metric_name, call_url, call_count,
class_name, method_name, params, children
]
def connect_url(action, host=None, with_port=False):
"""
:param action: the uploader actually intention
:return: ruled url, the suitable url according to headquarters strategy
"""
settings = global_settings()
scheme = settings.ssl and 'https' or 'http'
url_reg = "%s://%s/%s"
if host is None:
host = settings.host
url = url_reg % (scheme, host, action)
if with_port and settings.port:
url_reg = "%s://%s:%s/%s"
url = url_reg % (scheme, host, settings.port, action)
return url
def parse_proxy():
"""
:return: {'http': 'xxx', 'https': 'xxx'}
"""
settings = global_settings()
# Proxy schema set in configure file,then use it
# Else use `ssl` configure to decide 'http' or 'https'
host = settings.proxy_host or ''
port = settings.proxy_port or ''
user = settings.proxy_user or ''
scheme = settings.proxy_scheme or 'http'
password = settings.proxy_pwd or ''
# we support `http://user:password@host` format, so just checkout host
if not host:
return
# urlparse will return <scheme>://<netloc>/<path>
# (scheme, netloc, path, params, query, fragment)
sections = urlparse.urlparse(host)
if (not sections.scheme or sections.scheme not in ('http', 'https')) and not port:
return
path = ''
netloc = host
if sections.scheme: # config the host with <scheme>://<netloc>/<path> in settings
scheme = sections.scheme
netloc = sections.netloc
path = sections.path
elif sections.path: # urlparse parse the netloc to path
netloc = sections.path
if port:
netloc = "%s:%s" % (netloc, port)
if user:
if password:
netloc = '%s:%s@%s' % (user, password, netloc)
else:
netloc = '%s@%s' % (user, netloc)
full_url = '%s://%s%s' % (scheme, netloc, path)
return {'http': full_url, 'https': full_url}
def singleton_instance(name):
if not name:
name = global_settings().app_name
names = str(name).split(';')
app_name = names[0]
link_name = names[1:]
controller = dispatcher_instance()
instance = Proxy._instances.get(app_name, None)
if not instance:
with Proxy._lock:
instance = Proxy._instances.get(app_name, None)
if not instance:
instance = Proxy(controller, app_name, link_name)
Proxy._instances[app_name] = instance
console.info("Create new proxy with application name: %s",
name)
return instance
def connect_to_headquarters(self):
"""Performs the actual registration of the application to server, get server config and set the current app settings.
:return:
"""
# ensure the main thread get to run first
time.sleep(0.01)
while not self._active_session:
self.retry_upload_times = 0
settings = global_settings()
try:
active_session = create_connection(None, self._app_name, self._linked_app, env_config(), settings)
except InvalidLicenseException as _:
console.warning("Invalid license in configuration, agent will stop to work please fix license and"
"restart agent again")
self._is_license_valid = False
self._connect_event.wait(self.connect_retry_time)
continue
except Exception:
# use the harvest controller signal to control the connection
console.exception("Connect to agent server failed, Connection will try again in 1 min.")
self._connect_event.wait(self.connect_retry_time)
continue
finally:
if self._connect_event.isSet():
console.info("Agent is shutting down, stop the connection to server now.")
return
if active_session:
self._is_license_valid = True
self._connect_event.set()
self.start_sampler()
# set the application settings to data engine
with self._packets_lock:
self._active_session = active_session
self._packager.reset_packets(self._active_session.config)
def global_settings(self):
return global_settings()
def global_settings(self):
"""
:return:
"""
return global_settings()
def transmitter(session, url, action, payload={}, param={}, audit_mode=False):
"""
:param session: the request session to server
:param url: the address witch data send to
:param action: the send actually intention
:param payload: request data
:return: None
"""
console.debug("Send request with url %s, action %s param %s", url, action, param)
settings = global_settings()
start_time = time.time()
headers = {
"User-Agent": USER_AGENT,
"Content-Type": "application/octet-stream",
"connection": "close",
"Accept-Encoding": "deflate",
}
try:
data = json.dumps(payload)
except Exception as err:
console.error("Encoding json for payload failed, url %s action %s param %s payload %s err %s",
url, action, param, payload, err)
raise DiscardDataForRequest(str(sys.exc_info()[1]))
if len(data) > COMPRESS_MINIMUM_SIZE:
headers['Content-Encoding'] = 'deflate'
level = (len(data) < 2000000) and 1 or 9
data = zlib.compress(six.b(data), level)
auto_close_session = False
if not session:
session = requests.session()
auto_close_session = True
content = ""
try:
# because of the root license use the sha1 encryption, the version of the certifi new than 2015.4.28
# which used sha256 encryption, will cause the ssl verify failed, so no mater what kind of agent work
# mode(private deploy or public use), we do not verify the ssl license now.
verify_ssl = False if settings.port else settings.verify_certification
timeout = settings.data_report_timeout
proxy = parse_proxy()
print("->Send request with action %s" % action)
print("--->url %s" % url)
print("--->data %s" % data)
print("--->params %s" % param)
print("--->headers %s" % headers)
print("--->timeout %s" % timeout)
print("--->verify %s" % verify_ssl)
print("--->proxies %s" % proxy)
ret = session.post(url, data=data, params=param, headers=headers, timeout=timeout, verify=verify_ssl,
proxies=proxy)
content = ret.content
except requests.RequestException:
console.error('Agent server is not attachable. if the error continues, please report to networkbench.'
'The error raised was %s. response content is %s.', sys.exc_info()[1], content)
raise RetryDataForRequest(str(sys.exc_info()[1]))
finally:
duration = time.time() - start_time
if auto_close_session:
session.close()
if audit_mode:
console.info("Use %ss to upload data return value %r", duration, content)
if ret.status_code == 400:
console.error("Bad request has been submitted for url %s, please report this to us, thank u.", url)
raise DiscardDataForRequest()
elif ret.status_code == 503:
console.error("Agent server is unavailable. This can be a transient issue because of the server or our core"
" application being restarted. if this error continues, please report to us. thank u.")
raise DiscardDataForRequest()
elif ret.status_code == 502:
console.error("Agent server error, our engineer has caution this error, thanks for your support")
raise ServerIsUnavailable("service unavailable, get status code 502")
elif ret.status_code != 200:
console.warning("We got none 200 status code %s, this maybe some network/server error, if this error continues,"
"please report to us . thanks for your support. return content %s", ret.status_code, ret)
raise DiscardDataForRequest()
try:
if six.PY3:
content = content.decode('UTF-8')
print("<-Receive response: [%s]" % content)
result = json.loads(content)
except Exception as err:
console.warning("Decoding data for Json error. please contact us for further investigation. %s", err)
raise DiscardDataForRequest(str(sys.exc_info()[1]))
else:
# successful exchange with server
if result["status"] == "success":
return result["result"] if "result" in result else []
console.info("get unexpected return, there maybe some issue. %s", result)
server_status = int(result["result"]["errorCode"])
if server_status == 460:
console.warning("Invalid license key, Please contact to networkbench for more help.")
raise InvalidLicenseException("Invalid license key")
elif server_status == 462:
console.warning("Invalid data format, maybe something get wrong when json encoding.")
raise DiscardDataForRequest(content)
elif server_status == 461:
console.warning("Invalid data token, if this error continues, please report to networkbench support for further"
" investigation")
raise InvalidDataTokenException()
elif server_status == -1:
console.warning("Agent server error, our engineer has caution this error, thanks for your support.")
raise ServerIsUnavailable()
elif server_status == 470:
console.info("Configuration is out of date, server configuration will be obtain again")
raise OutOfDateConfigException()
return []
try:
import urlparse
except ImportError:
import urllib.parse as urlparse
from ssa.logistics.exceptions import DiscardDataForRequest, RetryDataForRequest, InvalidLicenseException
from ssa.logistics.exceptions import InvalidDataTokenException, OutOfDateConfigException, ServerIsUnavailable
from ssa.config.settings import global_settings
from ssa.packages import six, requests
console = logging.getLogger(__name__)
# 10kbi
COMPRESS_MINIMUM_SIZE = 10 * 1024
USER_AGENT = ' NBS Newlens Agent/%s (python %s; %s)' % (global_settings().agent_version, sys.version.split()[0],
sys.platform)
def parse_proxy():
"""
:return: {'http': 'xxx', 'https': 'xxx'}
"""
settings = global_settings()
# Proxy schema set in configure file,then use it
# Else use `ssl` configure to decide 'http' or 'https'
host = settings.proxy_host or ''
port = settings.proxy_port or ''
user = settings.proxy_user or ''
scheme = settings.proxy_scheme or 'http'
