def detect_connection(module): """ :param module: :return: """ if hasattr(module, 'Connection') and hasattr(module.Connection, 'connect'): wrap_function_trace(module, "Connection.connect", name="connect")
def detect_template_loader(module): """ :param module: :return: """ if hasattr(module, 'BaseLoader.load'): wrap_function_trace(module, 'BaseLoader.load')
def detect_mongo_client(module): """ :param module: :return: """ if hasattr(module, "MongoClient"): wrap_function_trace(module, "MongoClient.__init__", name="%s.MongoClient.__init__" % module.__name__)
def detect_connection(module): """ :param module: :return: """ if hasattr(module, "Connection"): wrap_function_trace(module, 'Connection.__init__', name='%s:Connection.__init__' % module.__name__)
def detect_template(module): """ :param module: :return: """ def get_template_name(filename, *args, **kwargs): """ """ return filename wrap_function_trace(module, 'parse_template', get_template_name, group='template.compile')
def detect_app_template(module): """ :param module: :return: """ def template_name(instance, name): """ :param instance: the instance of the Render :return: the template name """ return name wrap_function_trace(module, "Render.__getattr__", name=template_name, group='Template.Render') wrap_function_trace(module, "Template.compile_template", name=template_name, group='Template.compile')
def detect_app_entrance(module): """ :param module: :return: """ # trace the views function call metric wrap_function_wrapper(module.Flask, 'add_url_rule', add_url_rule_wrapper) wrap_function_wrapper(module.Flask, 'handle_exception', handle_exception_wrapper) # new feature form flask 0.7 if hasattr(module.Flask, 'handle_user_exception'): wrap_function_wrapper(module.Flask, 'handle_user_exception', handle_exception_wrapper) # new feature from flask 0.7 # todo: some issue fix if hasattr(module.Flask, 'full_dispatch_request'): wrap_function_trace(module.Flask, 'full_dispatch_request') # new feature from flask 0.7. this will occurred when errorhandler or register_error_handler is used by application if hasattr(module.Flask, '_register_error_handler'): wrap_function_wrapper(module.Flask, '_register_error_handler', wrap_register_error) # deal for before_first_request, new feature from 0.9 if hasattr(module.Flask, 'before_first_request'): wrap_function_wrapper(module.Flask, 'before_first_request', wrap_func_with_request) # deal for before_request if hasattr(module.Flask, 'before_request'): wrap_function_wrapper(module.Flask, 'before_request', wrap_func_with_request) # deal for after_request if hasattr(module.Flask, 'after_request'): wrap_function_wrapper(module.Flask, 'after_request', wrap_func_with_request) # deal for teardown_request if hasattr(module.Flask, 'teardown_request'): wrap_function_wrapper(module.Flask, 'teardown_request', wrap_func_with_request)
def detect_jinja2(module): """ """ def parse_tempate_name(instance, *args, **kwargs): return instance.name or instance.filename wrap_function_trace(module, 'Template.render', parse_tempate_name, 'Template.Render') def parse_template_for_env(instance, source, name=None, *args, **kwargs): """ :param instance: :param args: :param kwargs: :return: """ return name or 'template' wrap_function_trace(module, 'Environment.compile', parse_template_for_env, 'Template.compile')
def detect_templates(module): """ :param module: :return: """ # detect the template render if hasattr(module, 'SimpleTemplate'): wrap_function_trace(module, 'SimpleTemplate.render') if hasattr(module, 'MakoTemplate'): wrap_function_trace(module, 'MakoTemplate.render') if hasattr(module, 'CheetahTemplate'): wrap_function_trace(module, 'CheetahTemplate.render') if hasattr(module, 'Jinja2Template'): wrap_function_trace(module, 'Jinja2Template.render') if hasattr(module, 'SimpleTALTemplate'): wrap_function_trace(module, 'SimpleTALTemplate.render')
def detect_compileapp(module): """ :param module: :return: """ def get_name_in_controller(controller, function, environment): """more detail in gluon.compileapp.run_controller_in """ return '%s.%s' % (controller, function) def get_name_in_model(environment): """more detail in gluon.compileapp.run_model_in """ return '%s.%s' % (environment['request'].controller, environment['request'].function) def get_name_in_view(environment): """more detail in gluon.compileapp.run-view-in """ return '%s.%s' % (environment['request'].controller, environment['request'].function) wrap_function_trace(module, 'run_controller_in', get_name_in_controller, group='web2py.controllers') wrap_function_trace(module, 'run_models_in', get_name_in_model, group='web2py.models') wrap_function_trace(module, 'run_view_in', get_name_in_view, group='web2py.views')
def detect_core_mail(module): """ :param module: :return: """ wrap_function_trace(module, 'mail_admins') wrap_function_trace(module, 'mail_managers') wrap_function_trace(module, 'send_mail')
def trace_template_block_render(module, object_path): """ wrap the django template load tags times. :param module: :param object_path: :return: """ def block_name(instance, *args, **kwargs): """ The following params in django.template.loader_tags.BlockNode.render :param instance: :param args: :param kwargs: :return: """ return instance.name or "default-template" return wrap_function_trace(module, object_path, name=block_name, group='Template.Block')
def trace_django_template(module, object_path): """ :param modules: :param object_path: :return: """ def template_name(instance, *args, **kwargs): """ The following params in django.template.base.Template._render/Template.render :param instance: :param args: :param kwargs: :return: """ return instance.name or "default-template" return wrap_function_trace(module, object_path, name=template_name, group='Template/Render')
def trace_view_dispatch(module, object_path): """django view dispatch trace entrance. :param module: :param object_path: :return: """ def view_dispatch_name(instance, request, *args, **kwargs): """The following params in django.views.generic.base.View.dispatch :param instance: :param request: :param args: :param kwargs: :return: """ if request.method.lower() in instance.http_method_names: handler = getattr(instance, request.method.lower(), instance.http_method_not_allowed) else: handler = instance.http_method_not_allowed name = callable_name(handler) return name return wrap_function_trace(module, object_path, name=view_dispatch_name)
def detect_template(module): """ :param module: :return: """ def template_name(template, text, filename, *args): """ :param template: :param text: :param filename: :param args: compatible with _compile_text and _compile_module_file :return: """ return filename def template_name_in_render(instance, *args, **kwargs): return getattr(instance, 'filename', "template") wrap_function_trace(module, '_compile_text', name=template_name, group='Template.compile') wrap_function_trace(module, '_compile_module_file', name=template_name, group='Template.compile') wrap_function_trace(module, 'Template.render', name=template_name_in_render, group='Template.render')
def detect_channel(module): """ :return: """ # 基于selection connection libevent connect tornado connection 的生产者目前使用场景大部分为非web过程内 # 需要单独处理才能支持,咱不考虑 wrap_object(module, "Channel.basic_publish", mq_produce_trace_wrapper) wrap_function_wrapper(module.Channel, "basic_consume", mq_consume_wrapper) # 下面的方法都是采用即时读写的方式跟mq通信,故可以直接采集之性能 wrap_function_trace(module.Channel, "basic_get", name='basic_get') wrap_function_trace(module.Channel, "basic_nack", name='basic_nack') wrap_function_trace(module.Channel, "exchange_bind", name='exchange_bind') wrap_function_trace(module.Channel, "exchange_declare", name='exchange_declare') wrap_function_trace(module.Channel, "exchange_delete", name='exchange_delete') wrap_function_trace(module.Channel, "queue_bind", name='queue_bind') wrap_function_trace(module.Channel, "queue_declare", name='queue_declare') wrap_function_trace(module.Channel, "queue_delete", name='queue_delete') # 其他常规用户可能执行的回调 # 为了减少性能开销,不捕获基于CallbackManager.add所有的回调函数 wrap_function_wrapper(module, "Channel.add_on_cancel_callback", mq_common_callback_wrapper) wrap_function_wrapper(module, "Channel.add_on_close_callback", mq_common_callback_wrapper) wrap_function_wrapper(module, "Channel.add_on_return_callback", mq_common_callback_wrapper)
def detect_templates(module): """ :param module: :return: """ wrap_function_trace(module, '_render')
def detect(module): """ more interface description about dbapi2 in: https://www.python.org/dev/peps/pep-0249/ :param module: :return: """ dbtype = db_name_module.get(getattr(module, "__name__", "DBAPI2")) def parse_connect_params(connect_params): """解析连接数据库的host、port信息,支持mysql-python、pymysql、oursql、cx_Oracle插件 根据数据库分类来分类获取port信息 mysql-python: connect(host, user, pwd, port, db) pymysql connect(host=None, port=0, database=None) oursql http://pythonhosted.org/oursql: oursql.connect(host='127.0.0.1', user='******', passwd='foobar', db='example', port=3307) :param connect_params: :return: """ host, port, db_name = 'Unknown', 'Unknown', 'Unknown' if not connect_params or len(connect_params) != 2: return _args = connect_params[0] or () _kwargs = connect_params[1] or {} if dbtype == DB_TYPE_MYSQL: # 遵循DBAPI2协议前5个参数都是host=None, user=None, password="", database=None, port=0 host, port = _kwargs.get('host'), _kwargs.get('port') db_name = _kwargs.get('database') or _kwargs.get('db') if not host: host = _args[0] if len(_args) >= 1 else "Unknown" if not port: port = _args[4] if len(_args) >= 5 else 3306 else: port = 3306 if not db_name: db_name = _args[3] if len(_args) >= 4 else "Unknown" elif dbtype == DB_TYPE_ORACLE: # cx_Oracle.connect('hr', 'hrpwd', 'localhost:1521/XE') 或者 # '(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle.db.com)(PORT=1521)))\ # (CONNECT_DATA=(SID=nbsdb)))' if 3 == len(_args): con_info = re.findall(r'^(.+?):(.+?)/(.+?)$', str(_args[2]), re.IGNORECASE) if not con_info: con_info = re.findall( r'^.*HOST=(.+?)\).*PORT=(.+?)\).*SID=(.+?)\).?', str(_args[2]), re.IGNORECASE) if con_info and 3 == len(con_info[0]): host, port, db_name = con_info[0] # 'hr/hrpwd@localhost:1521/XE' if 1 == len(_args): con_str = str(_args[0]) _host = con_str[con_str.find("@") + 1:con_str.find(":")] _port = con_str[con_str.find(":") + 1:con_str.rfind("/")] _db_name = con_str[con_str.rfind("/") + 1:] if _host and _port and _db_name: host, port, db_name = _host, _port, _db_name elif dbtype == DB_TYPE_POSTGRELSQL: # 链接方式doc: https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING # postgresql://localhost/mydb # postgresql://localhost:5432/mydb # dsn参数: host=localhost port=5432 dbname=mydb connect_timeout=10 # 关键字参数 # todo: 不支持以下链接方式: # 不完整的URI连接, postgresql:///mydb?host=localhost&port=5433 # IPV4, postgresql://[2001:db8::1234]/database # 环境变量方式: https://www.postgresql.org/docs/current/static/libpq-envars.html _host, _port, _db_name = None, None, None if not _kwargs or len(_kwargs) == 1: con_str = _args[0] if 1 == len( _args) and not _kwargs else _kwargs.get('dsn', "") # host=localhost port=5432 dbname=mydb connect_timeout=10 if "//" not in con_str: for kv in con_str.split(): k, v = kv.split('=') if k.lower() not in ["host", "port", "dbname"]: continue if 'host' == k.lower(): _host = v elif 'port' == k.lower(): _port = v elif 'dbname' == k.lower(): _db_name = v else: # postgresql://[user[:password]@][netloc][:port][/dbname][?param1=value1&...] con_str = _args[0] if 1 == len( _args) and not _kwargs else _kwargs.get('dsn', "") sections = urlparse.urlparse(con_str) if sections.netloc: if "@" in sections.netloc: host_port = sections.netloc.split("@")[1] else: host_port = sections.netloc parts = host_port.split(":") if len(parts) == 2: _host, _port = parts[0], parts[1] elif len(parts) == 1: _host, _port = parts[0], 5432 _db_names = sections.path.replace('/', '', 1).split('?') _db_name = _db_names[0] if len( _db_names) >= 1 else sections.path else: _host, _port = _kwargs.get("host"), _kwargs.get("port") or 5432 _db_name = _kwargs.get("database") or _kwargs.get("dbname") host, port, db_name = _host or 'Unknown', _port or 5432, _db_name or 'Unknown' elif dbtype == DB_TYPE_SQLSERVER: # pyodbc 包仅支持sqlServer数据的抓取,其他类型无法识别,也几乎没人会用这么复杂的包连接别的数据库 if len(_args) >= 1 and 'DSN' not in str(_args[0]).upper(): # 'DSN=test;PWD=password',如果用户使用了DSN,无法抓取该信息 # ('DRIVER={FreeTDS};SERVER=sqlserver.db.com; PORT=1433;DATABASE=python_test; # UID=leng;PWD=Windows2008',) # 仅支持表中的一种port,host写法,不区分大小写 _host = re.findall(r"SERVER=(.+?);", str(_args[0]), re.IGNORECASE) _port = re.findall(r"port=(.+?);", str(_args[0]), re.IGNORECASE) _db_name = re.findall(r"DATABASE=(.+?);", str(_args[0]), re.IGNORECASE) _host, _port = _host[0] if _host else None, _port[ 0] if _port else None _db_name = _db_name[0] if _db_name else None # 会出现'DRIVER={FreeTDS};SERVER=sqlserver.db.com,1433;DATABASE=python_test;UID=leng;PWD=Windows2008' if _host and _db_name and not _port: parts = str(_host).split(",") if 2 == len(parts): _host = parts[0].strip() _port = parts[1].strip() host, port, db_name = _host or 'Unknown', _port or 'Unknown', _db_name or 'Unknown' else: logging.debug("Not support database type for capture ip & port.") return host, port, db_name class TingYunCursor(object): def __init__(self, cursor, cursor_params=None, connect_params=None): """https://docs.python.org/2.7/reference/datamodel.html?highlight=object.__setattr__#object.__setattr__ :param cursor: :param cursor_params: :param connect_params: :return: """ object.__setattr__(self, 'cursor', cursor) object.__setattr__(self, 'cursor_params', cursor_params) object.__setattr__(self, 'connect_params', connect_params) host, port, db_name = parse_connect_params(connect_params) object.__setattr__(self, 'host', host) object.__setattr__(self, 'port', port) object.__setattr__(self, 'db_name', db_name) def __setattr__(self, name, value): setattr(self.cursor, name, value) def __getattr__(self, name): return getattr(self.cursor, name) def __iter__(self): return iter(self.cursor) def fetchone(self, *args, **kwargs): """we do not capture the metric of execute result. this is small time used :args: :kwargs: :return: """ tracker = current_tracker() with FunctionTracker(tracker, callable(self.cursor.fetchone)): return self.cursor.fetchone(*args, **kwargs) def fetchmany(self, *args, **kwargs): """we do not capture the metric of execute result. this is small time used :args: :kwargs: :return: """ tracker = current_tracker() with FunctionTracker(tracker, callable(self.cursor.fetchone)): return self.cursor.fetchmany(*args, **kwargs) def fetchall(self, *args, **kwargs): """this operation maybe spend more time. this is small time used and the sql was executed. we can not take it :args: :kwargs: :return: """ tracker = current_tracker() with FunctionTracker(tracker, callable(self.cursor.fetchone)): return self.cursor.fetchall(*args, **kwargs) def execute(self, sql, *args, **kwargs): """ :param sql: :param args: :param kwargs: :return: """ tracker = current_tracker() if not tracker: return self.cursor.execute(sql, *args, **kwargs) with DatabaseTracker(tracker, sql, dbtype, module, self.connect_params, self.cursor_params, (args, kwargs), host=self.host, port=self.port, db_name=self.db_name) as dt: try: return self.cursor.execute(sql, *args, **kwargs) except: dt.exception = tracker.record_exception(is_error=False, additional_msg=sql) raise def executemany(self, sql, *args, **kwargs): """ :param sql: :param args: :param kwargs: :return: """ tracker = current_tracker() if not tracker: return self.cursor.executemany(sql, *args, **kwargs) with DatabaseTracker(tracker, sql, dbtype, module, host=self.host, port=self.port, db_name=self.db_name) as dt: try: return self.cursor.executemany(sql, *args, **kwargs) except: dt.exception = tracker.record_exception(is_error=False, additional_msg=sql) raise def callproc(self, procname, *args, **kwargs): """ :param procname: :param args: :param kwargs: :return: """ tracker = current_tracker() if not tracker: return self.cursor.callproc(procname, *args, **kwargs) with DatabaseTracker(tracker, 'CALLPROC %s' % procname, dbtype, module, host=self.host, port=self.port, db_name=self.db_name) as dt: try: return self.cursor.callproc(procname, *args, **kwargs) except: dt.exception = tracker.record_exception(is_error=False) raise class TingYunConnection(object): def __init__(self, connection, params=None): """ :param connection: :param params: :return: """ object.__setattr__(self, 'connection', connection) object.__setattr__(self, 'params', params) host, port, db_name = parse_connect_params(params) object.__setattr__(self, 'host', host) object.__setattr__(self, 'port', port) object.__setattr__(self, 'db_name', db_name) def __setattr__(self, name, value): setattr(self.connection, name, value) def __getattr__(self, name): return getattr(self.connection, name) def close(self, *args, **kwargs): """ :param args: :param kwargs: :return: """ return self.connection.close(*args, **kwargs) def cursor(self, *args, **kwargs): """ :param args: :param kwargs: :return: """ return TingYunCursor(self.connection.cursor(*args, **kwargs), (args, kwargs), self.params) def commit(self): """ :return: """ tracker = current_tracker() if not tracker: return self.connection.commit() with DatabaseTracker(tracker, 'COMMIT', dbtype, module, host=self.host, port=self.port, db_name=self.db_name) as dt: try: return self.connection.commit() except: dt.exception = tracker.record_exception(is_error=False) raise def rollback(self): """ :return: """ tracker = current_tracker() if not tracker: return self.connection.rollback() with DatabaseTracker(tracker, 'ROLLBACK', dbtype, module, host=self.host, port=self.port, db_name=self.db_name) as dt: try: return self.connection.rollback() except: dt.exception = tracker.record_exception(is_error=False) raise class DatabaseWrapper(object): def __init__(self, connect): self.connect = connect self.connect_instance = None def __call__(self, *args, **kwargs): """when module call the connect method. the wrapper will callback __call__ instead. :param args: :param kwargs: :return: """ self.connect_instance = self.connect(*args, **kwargs) cxn = TingYunConnection(self.connect_instance, (args, kwargs)) object.__setattr__(cxn, '_sqla_unwrap', cxn.connection) return cxn # Check if module is already wrapped if hasattr(module, '_self_dbapi2_wrapped'): return wrap_function_trace(module, 'connect', name='%s.connect' % dbtype, group='Database') module.connect = DatabaseWrapper(module.connect) module._self_dbapi2_wrapped = True
def detect_core_mail_message(module): """ :param module: :return: """ wrap_function_trace(module, 'EmailMessage.send')
def detect_http_multipartparser(module): """ detect for file upload :param module: :return: """ wrap_function_trace(module, 'MultiPartParser.parse')