def __init__(self, config=None): """ """ console.info("Start python agent with version %s", global_settings().agent_version) self._last_harvest = 0.0 self._harvest_thread = threading.Thread(target=self._harvest_loop, name="TingYunHarvestThread") self._harvest_thread.setDaemon(True) # should take caution to the doc self._harvest_shutdown = threading.Event() self._config = config if config is not None else global_settings() self._lock = threading.Lock() self._process_shutdown = False self._applications = {} # used to store the actived applications self.__max_tracker = 5000 self.__data_sampler = [] if self._config.enabled: atexit.register(self._atexit_shutdown) # Register our exit event to the uwsgi shutdown hooks. if 'uwsgi' in sys.modules: import uwsgi uwsgi_original_atexit_callback = getattr(uwsgi, 'atexit', None) def uwsgi_atexit_callback(): self._atexit_shutdown() if uwsgi_original_atexit_callback: uwsgi_original_atexit_callback() uwsgi.atexit = uwsgi_atexit_callback
def singleton_instance(): """ :return: singleton instance """ if Dispatcher._instance: print("sunyan: second time, Dispatcher._instance is not None.") return Dispatcher._instance print("sunyan: first time, Dispatcher._instance is None.") instance = None with Dispatcher._instance_lock: if not Dispatcher._instance: instance = Dispatcher(global_settings()) Dispatcher._instance = instance console.info('Creating instance of agent dispatcher') if instance: # FIXME #instance.register_data_sampler(memory_usage_sampler, ('memory.usage',)) instance.register_data_sampler(cpu_usage_sampler, ('cpu.usage', )) instance.active_dispatcher() return instance
def __init__(self, dispatcher, app_name, link_app_name=None): self.enabled = True self._app_name = app_name self._link_app_name = link_app_name if link_app_name else [] if not app_name: self._app_name = global_settings().app_name if not dispatcher: dispatcher = dispatcher_instance() console.debug("init application with new dispatcher.") self._dispatcher = dispatcher
def __init__(self, url, config, license_key, redirect_host): self.url = url self.config = config self.license_key = license_key self.local_setting = global_settings() self.merged_settings = config self.redirect_host = redirect_host self._session = None self.request_param = { "licenseKey": self.license_key, "version": self.local_setting.data_version, "appSessionKey": self.config.appSessionKey, }
def __init__(self, config_file): self.config_file = config_file self.is_embattled = False self.valid_embattle = True self._config_parser = ConfigParser.RawConfigParser() self._settings = global_settings() self._inspect_lock = threading.Lock() if not config_file: self.config_file = os.environ.get(ENV_CONFIG_FILE, None) if not self.config_file: log_bootstrap('Agent config file is not found, agent start failed.') self.valid_embattle = False log_bootstrap("get config file %s" % self.config_file)
def trace_node(self, root): """ :param root: the root node of the tracker :return: """ dbtype = r" %s" % self.dbtype params = {"sql": "", "explainPlan": {}, "stacktrace": []} children = [] call_count = 1 class_name = "" method_name = root.name root.trace_node_count += 1 start_time = node_start_time(root, self) end_time = node_end_time(root, self) operation = str(self.operation).upper() metric_name = "Database%s/CALL/sql" % dbtype call_url = metric_name if operation in ('SELECT', 'UPDATE', 'INSERT', 'DELETE'): metric_name = 'Database/%s/%s' % ( self.table, operation) if self.table else 'Database/NULL/%s' % operation if self.formatted: # Note, use local setting only. _settings = global_settings() params['sql'] = self.formatted if _settings.action_tracer.log_sql: console.info( "Log sql is opened. sql upload is disabled, sql sentence is %s", self.formatted) params['sql'] = "" if self.explain_plan: params['explainPlan'] = self.explain_plan if self.stack_trace: for line in self.stack_trace: if len(line) >= 4 and 'tingyun' not in line[0]: params['stacktrace'].append( "%s(%s:%s)" % (line[2], line[0], line[1])) return [ start_time, end_time, metric_name, call_url, call_count, class_name, method_name, params, children ]
def connect_url(action, host=None, with_port=False): """ :param action: the uploader actually intention :return: ruled url, the suitable url according to headquarters strategy """ settings = global_settings() scheme = settings.ssl and 'https' or 'http' url_reg = "%s://%s/%s" if host is None: host = settings.host url = url_reg % (scheme, host, action) if with_port and settings.port: url_reg = "%s://%s:%s/%s" url = url_reg % (scheme, host, settings.port, action) return url
def parse_proxy(): """ :return: {'http': 'xxx', 'https': 'xxx'} """ settings = global_settings() # Proxy schema set in configure file,then use it # Else use `ssl` configure to decide 'http' or 'https' host = settings.proxy_host or '' port = settings.proxy_port or '' user = settings.proxy_user or '' scheme = settings.proxy_scheme or 'http' password = settings.proxy_pwd or '' # we support `http://user:password@host` format, so just checkout host if not host: return # urlparse will return <scheme>://<netloc>/<path> # (scheme, netloc, path, params, query, fragment) sections = urlparse.urlparse(host) if (not sections.scheme or sections.scheme not in ('http', 'https')) and not port: return path = '' netloc = host if sections.scheme: # config the host with <scheme>://<netloc>/<path> in settings scheme = sections.scheme netloc = sections.netloc path = sections.path elif sections.path: # urlparse parse the netloc to path netloc = sections.path if port: netloc = "%s:%s" % (netloc, port) if user: if password: netloc = '%s:%s@%s' % (user, password, netloc) else: netloc = '%s@%s' % (user, netloc) full_url = '%s://%s%s' % (scheme, netloc, path) return {'http': full_url, 'https': full_url}
def singleton_instance(name): if not name: name = global_settings().app_name names = str(name).split(';') app_name = names[0] link_name = names[1:] controller = dispatcher_instance() instance = Proxy._instances.get(app_name, None) if not instance: with Proxy._lock: instance = Proxy._instances.get(app_name, None) if not instance: instance = Proxy(controller, app_name, link_name) Proxy._instances[app_name] = instance console.info("Create new proxy with application name: %s", name) return instance
def connect_to_headquarters(self): """Performs the actual registration of the application to server, get server config and set the current app settings. :return: """ # ensure the main thread get to run first time.sleep(0.01) while not self._active_session: self.retry_upload_times = 0 settings = global_settings() try: active_session = create_connection(None, self._app_name, self._linked_app, env_config(), settings) except InvalidLicenseException as _: console.warning("Invalid license in configuration, agent will stop to work please fix license and" "restart agent again") self._is_license_valid = False self._connect_event.wait(self.connect_retry_time) continue except Exception: # use the harvest controller signal to control the connection console.exception("Connect to agent server failed, Connection will try again in 1 min.") self._connect_event.wait(self.connect_retry_time) continue finally: if self._connect_event.isSet(): console.info("Agent is shutting down, stop the connection to server now.") return if active_session: self._is_license_valid = True self._connect_event.set() self.start_sampler() # set the application settings to data engine with self._packets_lock: self._active_session = active_session self._packager.reset_packets(self._active_session.config)
def global_settings(self): return global_settings()
def global_settings(self): """ :return: """ return global_settings()
def transmitter(session, url, action, payload={}, param={}, audit_mode=False): """ :param session: the request session to server :param url: the address witch data send to :param action: the send actually intention :param payload: request data :return: None """ console.debug("Send request with url %s, action %s param %s", url, action, param) settings = global_settings() start_time = time.time() headers = { "User-Agent": USER_AGENT, "Content-Type": "application/octet-stream", "connection": "close", "Accept-Encoding": "deflate", } try: data = json.dumps(payload) except Exception as err: console.error("Encoding json for payload failed, url %s action %s param %s payload %s err %s", url, action, param, payload, err) raise DiscardDataForRequest(str(sys.exc_info()[1])) if len(data) > COMPRESS_MINIMUM_SIZE: headers['Content-Encoding'] = 'deflate' level = (len(data) < 2000000) and 1 or 9 data = zlib.compress(six.b(data), level) auto_close_session = False if not session: session = requests.session() auto_close_session = True content = "" try: # because of the root license use the sha1 encryption, the version of the certifi new than 2015.4.28 # which used sha256 encryption, will cause the ssl verify failed, so no mater what kind of agent work # mode(private deploy or public use), we do not verify the ssl license now. verify_ssl = False if settings.port else settings.verify_certification timeout = settings.data_report_timeout proxy = parse_proxy() print("->Send request with action %s" % action) print("--->url %s" % url) print("--->data %s" % data) print("--->params %s" % param) print("--->headers %s" % headers) print("--->timeout %s" % timeout) print("--->verify %s" % verify_ssl) print("--->proxies %s" % proxy) ret = session.post(url, data=data, params=param, headers=headers, timeout=timeout, verify=verify_ssl, proxies=proxy) content = ret.content except requests.RequestException: console.error('Agent server is not attachable. if the error continues, please report to networkbench.' 'The error raised was %s. response content is %s.', sys.exc_info()[1], content) raise RetryDataForRequest(str(sys.exc_info()[1])) finally: duration = time.time() - start_time if auto_close_session: session.close() if audit_mode: console.info("Use %ss to upload data return value %r", duration, content) if ret.status_code == 400: console.error("Bad request has been submitted for url %s, please report this to us, thank u.", url) raise DiscardDataForRequest() elif ret.status_code == 503: console.error("Agent server is unavailable. This can be a transient issue because of the server or our core" " application being restarted. if this error continues, please report to us. thank u.") raise DiscardDataForRequest() elif ret.status_code == 502: console.error("Agent server error, our engineer has caution this error, thanks for your support") raise ServerIsUnavailable("service unavailable, get status code 502") elif ret.status_code != 200: console.warning("We got none 200 status code %s, this maybe some network/server error, if this error continues," "please report to us . thanks for your support. return content %s", ret.status_code, ret) raise DiscardDataForRequest() try: if six.PY3: content = content.decode('UTF-8') print("<-Receive response: [%s]" % content) result = json.loads(content) except Exception as err: console.warning("Decoding data for Json error. please contact us for further investigation. %s", err) raise DiscardDataForRequest(str(sys.exc_info()[1])) else: # successful exchange with server if result["status"] == "success": return result["result"] if "result" in result else [] console.info("get unexpected return, there maybe some issue. %s", result) server_status = int(result["result"]["errorCode"]) if server_status == 460: console.warning("Invalid license key, Please contact to networkbench for more help.") raise InvalidLicenseException("Invalid license key") elif server_status == 462: console.warning("Invalid data format, maybe something get wrong when json encoding.") raise DiscardDataForRequest(content) elif server_status == 461: console.warning("Invalid data token, if this error continues, please report to networkbench support for further" " investigation") raise InvalidDataTokenException() elif server_status == -1: console.warning("Agent server error, our engineer has caution this error, thanks for your support.") raise ServerIsUnavailable() elif server_status == 470: console.info("Configuration is out of date, server configuration will be obtain again") raise OutOfDateConfigException() return []
try: import urlparse except ImportError: import urllib.parse as urlparse from ssa.logistics.exceptions import DiscardDataForRequest, RetryDataForRequest, InvalidLicenseException from ssa.logistics.exceptions import InvalidDataTokenException, OutOfDateConfigException, ServerIsUnavailable from ssa.config.settings import global_settings from ssa.packages import six, requests console = logging.getLogger(__name__) # 10kbi COMPRESS_MINIMUM_SIZE = 10 * 1024 USER_AGENT = ' NBS Newlens Agent/%s (python %s; %s)' % (global_settings().agent_version, sys.version.split()[0], sys.platform) def parse_proxy(): """ :return: {'http': 'xxx', 'https': 'xxx'} """ settings = global_settings() # Proxy schema set in configure file,then use it # Else use `ssl` configure to decide 'http' or 'https' host = settings.proxy_host or '' port = settings.proxy_port or '' user = settings.proxy_user or '' scheme = settings.proxy_scheme or 'http'