def detect_connection(module):
"""
:param module:
:return:
"""
if hasattr(module, 'Connection') and hasattr(module.Connection, 'connect'):
wrap_function_trace(module, "Connection.connect", name="connect")
def detect_template_loader(module):
"""
:param module:
:return:
"""
if hasattr(module, 'BaseLoader.load'):
wrap_function_trace(module, 'BaseLoader.load')
def detect_mongo_client(module):
"""
:param module:
:return:
"""
if hasattr(module, "MongoClient"):
wrap_function_trace(module,
"MongoClient.__init__",
name="%s.MongoClient.__init__" % module.__name__)
def detect_connection(module):
"""
:param module:
:return:
"""
if hasattr(module, "Connection"):
wrap_function_trace(module,
'Connection.__init__',
name='%s:Connection.__init__' % module.__name__)
def detect_template(module):
"""
:param module:
:return:
"""
def get_template_name(filename, *args, **kwargs):
"""
"""
return filename
wrap_function_trace(module,
'parse_template',
get_template_name,
group='template.compile')
def detect_app_template(module):
"""
:param module:
:return:
"""
def template_name(instance, name):
"""
:param instance: the instance of the Render
:return: the template name
"""
return name
wrap_function_trace(module, "Render.__getattr__", name=template_name, group='Template.Render')
wrap_function_trace(module, "Template.compile_template", name=template_name, group='Template.compile')
def detect_app_entrance(module):
"""
:param module:
:return:
"""
# trace the views function call metric
wrap_function_wrapper(module.Flask, 'add_url_rule', add_url_rule_wrapper)
wrap_function_wrapper(module.Flask, 'handle_exception',
handle_exception_wrapper)
# new feature form flask 0.7
if hasattr(module.Flask, 'handle_user_exception'):
wrap_function_wrapper(module.Flask, 'handle_user_exception',
handle_exception_wrapper)
# new feature from flask 0.7
# todo: some issue fix
if hasattr(module.Flask, 'full_dispatch_request'):
wrap_function_trace(module.Flask, 'full_dispatch_request')
# new feature from flask 0.7. this will occurred when errorhandler or register_error_handler is used by application
if hasattr(module.Flask, '_register_error_handler'):
wrap_function_wrapper(module.Flask, '_register_error_handler',
wrap_register_error)
# deal for before_first_request, new feature from 0.9
if hasattr(module.Flask, 'before_first_request'):
wrap_function_wrapper(module.Flask, 'before_first_request',
wrap_func_with_request)
# deal for before_request
if hasattr(module.Flask, 'before_request'):
wrap_function_wrapper(module.Flask, 'before_request',
wrap_func_with_request)
# deal for after_request
if hasattr(module.Flask, 'after_request'):
wrap_function_wrapper(module.Flask, 'after_request',
wrap_func_with_request)
# deal for teardown_request
if hasattr(module.Flask, 'teardown_request'):
wrap_function_wrapper(module.Flask, 'teardown_request',
wrap_func_with_request)
def detect_jinja2(module):
"""
"""
def parse_tempate_name(instance, *args, **kwargs):
return instance.name or instance.filename
wrap_function_trace(module, 'Template.render', parse_tempate_name,
'Template.Render')
def parse_template_for_env(instance, source, name=None, *args, **kwargs):
"""
:param instance:
:param args:
:param kwargs:
:return:
"""
return name or 'template'
wrap_function_trace(module, 'Environment.compile', parse_template_for_env,
'Template.compile')
def detect_templates(module):
"""
:param module:
:return:
"""
# detect the template render
if hasattr(module, 'SimpleTemplate'):
wrap_function_trace(module, 'SimpleTemplate.render')
if hasattr(module, 'MakoTemplate'):
wrap_function_trace(module, 'MakoTemplate.render')
if hasattr(module, 'CheetahTemplate'):
wrap_function_trace(module, 'CheetahTemplate.render')
if hasattr(module, 'Jinja2Template'):
wrap_function_trace(module, 'Jinja2Template.render')
if hasattr(module, 'SimpleTALTemplate'):
wrap_function_trace(module, 'SimpleTALTemplate.render')
def detect_compileapp(module):
"""
:param module:
:return:
"""
def get_name_in_controller(controller, function, environment):
"""more detail in gluon.compileapp.run_controller_in
"""
return '%s.%s' % (controller, function)
def get_name_in_model(environment):
"""more detail in gluon.compileapp.run_model_in
"""
return '%s.%s' % (environment['request'].controller,
environment['request'].function)
def get_name_in_view(environment):
"""more detail in gluon.compileapp.run-view-in
"""
return '%s.%s' % (environment['request'].controller,
environment['request'].function)
wrap_function_trace(module,
'run_controller_in',
get_name_in_controller,
group='web2py.controllers')
wrap_function_trace(module,
'run_models_in',
get_name_in_model,
group='web2py.models')
wrap_function_trace(module,
'run_view_in',
get_name_in_view,
group='web2py.views')
def detect_core_mail(module):
"""
:param module:
:return:
"""
wrap_function_trace(module, 'mail_admins')
wrap_function_trace(module, 'mail_managers')
wrap_function_trace(module, 'send_mail')
def trace_template_block_render(module, object_path):
""" wrap the django template load tags times.
:param module:
:param object_path:
:return:
"""
def block_name(instance, *args, **kwargs):
""" The following params in django.template.loader_tags.BlockNode.render
:param instance:
:param args:
:param kwargs:
:return:
"""
return instance.name or "default-template"
return wrap_function_trace(module,
object_path,
name=block_name,
group='Template.Block')
def trace_django_template(module, object_path):
"""
:param modules:
:param object_path:
:return:
"""
def template_name(instance, *args, **kwargs):
""" The following params in django.template.base.Template._render/Template.render
:param instance:
:param args:
:param kwargs:
:return:
"""
return instance.name or "default-template"
return wrap_function_trace(module,
object_path,
name=template_name,
group='Template/Render')
def trace_view_dispatch(module, object_path):
"""django view dispatch trace entrance.
:param module:
:param object_path:
:return:
"""
def view_dispatch_name(instance, request, *args, **kwargs):
"""The following params in django.views.generic.base.View.dispatch
:param instance:
:param request:
:param args:
:param kwargs:
:return:
"""
if request.method.lower() in instance.http_method_names:
handler = getattr(instance, request.method.lower(),
instance.http_method_not_allowed)
else:
handler = instance.http_method_not_allowed
name = callable_name(handler)
return name
return wrap_function_trace(module, object_path, name=view_dispatch_name)
def detect_template(module):
"""
:param module:
:return:
"""
def template_name(template, text, filename, *args):
"""
:param template:
:param text:
:param filename:
:param args: compatible with _compile_text and _compile_module_file
:return:
"""
return filename
def template_name_in_render(instance, *args, **kwargs):
return getattr(instance, 'filename', "template")
wrap_function_trace(module, '_compile_text', name=template_name, group='Template.compile')
wrap_function_trace(module, '_compile_module_file', name=template_name, group='Template.compile')
wrap_function_trace(module, 'Template.render', name=template_name_in_render, group='Template.render')
def detect_channel(module):
"""
:return:
"""
# 基于selection connection libevent connect tornado connection 的生产者目前使用场景大部分为非web过程内
# 需要单独处理才能支持，咱不考虑
wrap_object(module, "Channel.basic_publish", mq_produce_trace_wrapper)
wrap_function_wrapper(module.Channel, "basic_consume", mq_consume_wrapper)
# 下面的方法都是采用即时读写的方式跟mq通信，故可以直接采集之性能
wrap_function_trace(module.Channel, "basic_get", name='basic_get')
wrap_function_trace(module.Channel, "basic_nack", name='basic_nack')
wrap_function_trace(module.Channel, "exchange_bind", name='exchange_bind')
wrap_function_trace(module.Channel, "exchange_declare", name='exchange_declare')
wrap_function_trace(module.Channel, "exchange_delete", name='exchange_delete')
wrap_function_trace(module.Channel, "queue_bind", name='queue_bind')
wrap_function_trace(module.Channel, "queue_declare", name='queue_declare')
wrap_function_trace(module.Channel, "queue_delete", name='queue_delete')
# 其他常规用户可能执行的回调
# 为了减少性能开销，不捕获基于CallbackManager.add所有的回调函数
wrap_function_wrapper(module, "Channel.add_on_cancel_callback", mq_common_callback_wrapper)
wrap_function_wrapper(module, "Channel.add_on_close_callback", mq_common_callback_wrapper)
wrap_function_wrapper(module, "Channel.add_on_return_callback", mq_common_callback_wrapper)
def detect_templates(module):
"""
:param module:
:return:
"""
wrap_function_trace(module, '_render')
def detect(module):
""" more interface description about dbapi2 in: https://www.python.org/dev/peps/pep-0249/
:param module:
:return:
"""
dbtype = db_name_module.get(getattr(module, "__name__", "DBAPI2"))
def parse_connect_params(connect_params):
"""解析连接数据库的host、port信息，支持mysql-python、pymysql、oursql、cx_Oracle插件
根据数据库分类来分类获取port信息
mysql-python:
connect(host, user, pwd, port, db)
pymysql
connect(host=None, port=0, database=None)
oursql http://pythonhosted.org/oursql：
oursql.connect(host='127.0.0.1', user='******', passwd='foobar',
db='example', port=3307)
:param connect_params:
:return:
"""
host, port, db_name = 'Unknown', 'Unknown', 'Unknown'
if not connect_params or len(connect_params) != 2:
return
_args = connect_params[0] or ()
_kwargs = connect_params[1] or {}
if dbtype == DB_TYPE_MYSQL:
# 遵循DBAPI2协议前5个参数都是host=None, user=None, password="", database=None, port=0
host, port = _kwargs.get('host'), _kwargs.get('port')
db_name = _kwargs.get('database') or _kwargs.get('db')
if not host:
host = _args[0] if len(_args) >= 1 else "Unknown"
if not port:
port = _args[4] if len(_args) >= 5 else 3306
else:
port = 3306
if not db_name:
db_name = _args[3] if len(_args) >= 4 else "Unknown"
elif dbtype == DB_TYPE_ORACLE:
# cx_Oracle.connect('hr', 'hrpwd', 'localhost:1521/XE') 或者
# '(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle.db.com)(PORT=1521)))\
# (CONNECT_DATA=(SID=nbsdb)))'
if 3 == len(_args):
con_info = re.findall(r'^(.+?):(.+?)/(.+?)$', str(_args[2]),
re.IGNORECASE)
if not con_info:
con_info = re.findall(
r'^.*HOST=(.+?)\).*PORT=(.+?)\).*SID=(.+?)\).?',
str(_args[2]), re.IGNORECASE)
if con_info and 3 == len(con_info[0]):
host, port, db_name = con_info[0]
# 'hr/hrpwd@localhost:1521/XE'
if 1 == len(_args):
con_str = str(_args[0])
_host = con_str[con_str.find("@") + 1:con_str.find(":")]
_port = con_str[con_str.find(":") + 1:con_str.rfind("/")]
_db_name = con_str[con_str.rfind("/") + 1:]
if _host and _port and _db_name:
host, port, db_name = _host, _port, _db_name
elif dbtype == DB_TYPE_POSTGRELSQL:
# 链接方式doc: https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
# postgresql://localhost/mydb
# postgresql://localhost:5432/mydb
# dsn参数: host=localhost port=5432 dbname=mydb connect_timeout=10
# 关键字参数
# todo: 不支持以下链接方式：
# 不完整的URI连接， postgresql:///mydb?host=localhost&port=5433
# IPV4, postgresql://[2001:db8::1234]/database
# 环境变量方式： https://www.postgresql.org/docs/current/static/libpq-envars.html
_host, _port, _db_name = None, None, None
if not _kwargs or len(_kwargs) == 1:
con_str = _args[0] if 1 == len(
_args) and not _kwargs else _kwargs.get('dsn', "")
# host=localhost port=5432 dbname=mydb connect_timeout=10
if "//" not in con_str:
for kv in con_str.split():
k, v = kv.split('=')
if k.lower() not in ["host", "port", "dbname"]:
continue
if 'host' == k.lower():
_host = v
elif 'port' == k.lower():
_port = v
elif 'dbname' == k.lower():
_db_name = v
else:
# postgresql://[user[:password]@][netloc][:port][/dbname][?param1=value1&...]
con_str = _args[0] if 1 == len(
_args) and not _kwargs else _kwargs.get('dsn', "")
sections = urlparse.urlparse(con_str)
if sections.netloc:
if "@" in sections.netloc:
host_port = sections.netloc.split("@")[1]
else:
host_port = sections.netloc
parts = host_port.split(":")
if len(parts) == 2:
_host, _port = parts[0], parts[1]
elif len(parts) == 1:
_host, _port = parts[0], 5432
_db_names = sections.path.replace('/', '',
1).split('?')
_db_name = _db_names[0] if len(
_db_names) >= 1 else sections.path
else:
_host, _port = _kwargs.get("host"), _kwargs.get("port") or 5432
_db_name = _kwargs.get("database") or _kwargs.get("dbname")
host, port, db_name = _host or 'Unknown', _port or 5432, _db_name or 'Unknown'
elif dbtype == DB_TYPE_SQLSERVER:
# pyodbc 包仅支持sqlServer数据的抓取，其他类型无法识别，也几乎没人会用这么复杂的包连接别的数据库
if len(_args) >= 1 and 'DSN' not in str(_args[0]).upper():
# 'DSN=test;PWD=password',如果用户使用了DSN，无法抓取该信息
# ('DRIVER={FreeTDS};SERVER=sqlserver.db.com; PORT=1433;DATABASE=python_test;
# UID=leng;PWD=Windows2008',)
# 仅支持表中的一种port，host写法,不区分大小写
_host = re.findall(r"SERVER=(.+?);", str(_args[0]),
re.IGNORECASE)
_port = re.findall(r"port=(.+?);", str(_args[0]),
re.IGNORECASE)
_db_name = re.findall(r"DATABASE=(.+?);", str(_args[0]),
re.IGNORECASE)
_host, _port = _host[0] if _host else None, _port[
0] if _port else None
_db_name = _db_name[0] if _db_name else None
# 会出现'DRIVER={FreeTDS};SERVER=sqlserver.db.com,1433;DATABASE=python_test;UID=leng;PWD=Windows2008'
if _host and _db_name and not _port:
parts = str(_host).split(",")
if 2 == len(parts):
_host = parts[0].strip()
_port = parts[1].strip()
host, port, db_name = _host or 'Unknown', _port or 'Unknown', _db_name or 'Unknown'
else:
logging.debug("Not support database type for capture ip & port.")
return host, port, db_name
class TingYunCursor(object):
def __init__(self, cursor, cursor_params=None, connect_params=None):
"""https://docs.python.org/2.7/reference/datamodel.html?highlight=object.__setattr__#object.__setattr__
:param cursor:
:param cursor_params:
:param connect_params:
:return:
"""
object.__setattr__(self, 'cursor', cursor)
object.__setattr__(self, 'cursor_params', cursor_params)
object.__setattr__(self, 'connect_params', connect_params)
host, port, db_name = parse_connect_params(connect_params)
object.__setattr__(self, 'host', host)
object.__setattr__(self, 'port', port)
object.__setattr__(self, 'db_name', db_name)
def __setattr__(self, name, value):
setattr(self.cursor, name, value)
def __getattr__(self, name):
return getattr(self.cursor, name)
def __iter__(self):
return iter(self.cursor)
def fetchone(self, *args, **kwargs):
"""we do not capture the metric of execute result. this is small time used
:args:
:kwargs:
:return:
"""
tracker = current_tracker()
with FunctionTracker(tracker, callable(self.cursor.fetchone)):
return self.cursor.fetchone(*args, **kwargs)
def fetchmany(self, *args, **kwargs):
"""we do not capture the metric of execute result. this is small time used
:args:
:kwargs:
:return:
"""
tracker = current_tracker()
with FunctionTracker(tracker, callable(self.cursor.fetchone)):
return self.cursor.fetchmany(*args, **kwargs)
def fetchall(self, *args, **kwargs):
"""this operation maybe spend more time. this is small time used
and the sql was executed. we can not take it
:args:
:kwargs:
:return:
"""
tracker = current_tracker()
with FunctionTracker(tracker, callable(self.cursor.fetchone)):
return self.cursor.fetchall(*args, **kwargs)
def execute(self, sql, *args, **kwargs):
"""
:param sql:
:param args:
:param kwargs:
:return:
"""
tracker = current_tracker()
if not tracker:
return self.cursor.execute(sql, *args, **kwargs)
with DatabaseTracker(tracker,
sql,
dbtype,
module,
self.connect_params,
self.cursor_params, (args, kwargs),
host=self.host,
port=self.port,
db_name=self.db_name) as dt:
try:
return self.cursor.execute(sql, *args, **kwargs)
except:
dt.exception = tracker.record_exception(is_error=False,
additional_msg=sql)
raise
def executemany(self, sql, *args, **kwargs):
"""
:param sql:
:param args:
:param kwargs:
:return:
"""
tracker = current_tracker()
if not tracker:
return self.cursor.executemany(sql, *args, **kwargs)
with DatabaseTracker(tracker,
sql,
dbtype,
module,
host=self.host,
port=self.port,
db_name=self.db_name) as dt:
try:
return self.cursor.executemany(sql, *args, **kwargs)
except:
dt.exception = tracker.record_exception(is_error=False,
additional_msg=sql)
raise
def callproc(self, procname, *args, **kwargs):
"""
:param procname:
:param args:
:param kwargs:
:return:
"""
tracker = current_tracker()
if not tracker:
return self.cursor.callproc(procname, *args, **kwargs)
with DatabaseTracker(tracker,
'CALLPROC %s' % procname,
dbtype,
module,
host=self.host,
port=self.port,
db_name=self.db_name) as dt:
try:
return self.cursor.callproc(procname, *args, **kwargs)
except:
dt.exception = tracker.record_exception(is_error=False)
raise
class TingYunConnection(object):
def __init__(self, connection, params=None):
"""
:param connection:
:param params:
:return:
"""
object.__setattr__(self, 'connection', connection)
object.__setattr__(self, 'params', params)
host, port, db_name = parse_connect_params(params)
object.__setattr__(self, 'host', host)
object.__setattr__(self, 'port', port)
object.__setattr__(self, 'db_name', db_name)
def __setattr__(self, name, value):
setattr(self.connection, name, value)
def __getattr__(self, name):
return getattr(self.connection, name)
def close(self, *args, **kwargs):
"""
:param args:
:param kwargs:
:return:
"""
return self.connection.close(*args, **kwargs)
def cursor(self, *args, **kwargs):
"""
:param args:
:param kwargs:
:return:
"""
return TingYunCursor(self.connection.cursor(*args, **kwargs),
(args, kwargs), self.params)
def commit(self):
"""
:return:
"""
tracker = current_tracker()
if not tracker:
return self.connection.commit()
with DatabaseTracker(tracker,
'COMMIT',
dbtype,
module,
host=self.host,
port=self.port,
db_name=self.db_name) as dt:
try:
return self.connection.commit()
except:
dt.exception = tracker.record_exception(is_error=False)
raise
def rollback(self):
"""
:return:
"""
tracker = current_tracker()
if not tracker:
return self.connection.rollback()
with DatabaseTracker(tracker,
'ROLLBACK',
dbtype,
module,
host=self.host,
port=self.port,
db_name=self.db_name) as dt:
try:
return self.connection.rollback()
except:
dt.exception = tracker.record_exception(is_error=False)
raise
class DatabaseWrapper(object):
def __init__(self, connect):
self.connect = connect
self.connect_instance = None
def __call__(self, *args, **kwargs):
"""when module call the connect method. the wrapper will callback __call__ instead.
:param args:
:param kwargs:
:return:
"""
self.connect_instance = self.connect(*args, **kwargs)
cxn = TingYunConnection(self.connect_instance, (args, kwargs))
object.__setattr__(cxn, '_sqla_unwrap', cxn.connection)
return cxn
# Check if module is already wrapped
if hasattr(module, '_self_dbapi2_wrapped'):
return
wrap_function_trace(module,
'connect',
name='%s.connect' % dbtype,
group='Database')
module.connect = DatabaseWrapper(module.connect)
module._self_dbapi2_wrapped = True
def detect_core_mail_message(module):
"""
:param module:
:return:
"""
wrap_function_trace(module, 'EmailMessage.send')
def detect_http_multipartparser(module):
""" detect for file upload
:param module:
:return:
"""
wrap_function_trace(module, 'MultiPartParser.parse')
