def log_func_exec(exec_time, func, args, kwargs, show_arg=None):
if show_arg is not None:
kwargs = {}
print_args = []
for i in xrange(0, len(args)):
if i in show_arg:
print_args.append(args[i])
else:
print_args = args
try:
logging_time = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")
clz_name = ''
co_varnames = func.func_code.co_varnames
if co_varnames and len(co_varnames) > 0 and co_varnames[0] == 'self':
# setattr(vv, meth_name, profile_aspect()(meth, vv))
try:
clz_name = args[0].__class__.__name__ + '.'
except:
pass
la = "{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}\t{8}".format(
profiler.project_id, logging_time, local_ip,
profile_local.__dict__.get('url', ''), round(exec_time * 1000, 3),
profile_local.__dict__.get('uuid', ''),
func.__module__ + "." + clz_name + func.func_name,
re.sub(spaces_pattern, ' ', str(print_args)),
re.sub(spaces_pattern, ' ', str(kwargs)))
kk_log.get_kk_log(topic_name).info(la)
except:
log.error("log_func_exec error: %s", utils.current_exception_info())
def reload_module_functions(self, current_funcs, new_funcs):
try:
if not does_local_ip_matched():
return
# in current_funcs, but not in new_funcs, need to restore profile
diff_restore = list(set(current_funcs).difference(set(new_funcs)))
for aa in diff_restore:
arr = str(aa).split(',')
patt = build_pattern(arr[0], arr[1])
if patt in self.function_wrappers:
fw = self.function_wrappers[patt]
fw.restore_function_aspect()
self.function_wrappers.pop(patt)
if aa in new_funcs:
new_funcs.remove(aa)
# not in current_funcs, but in new_funcs, need to add profile
# diff_profile = list(set(new_funcs).difference(set(current_funcs)))
for aa in new_funcs:
arr = str(aa).split(',')
self.profile_module_function(arr[0], arr[1])
except:
log.error('reload_module_functions %s %s error: %s',
str(current_funcs), str(new_funcs),
utils.current_exception_info())
def __find_sub_modules(self, work_dir, top_dir):
ww = work_dir + "/" + top_dir
if os.path.isfile(ww):
print 'ww is ', ww
self.__add_module_by_file(ww)
return
try:
for sub in os.listdir(ww):
ss = top_dir + '/' + sub
www = work_dir + '/' + ss
if os.path.isfile(www):
module_name = self.__add_module_by_file(ss)
mods = module_dir_names.get(top_dir, None)
if mods is None:
mods = list()
module_dir_names[top_dir] = mods
if module_name:
mods.append(module_name)
else:
# print 'work_dir is ', work_dir, ' subdir is ', ss
self.__find_sub_modules(work_dir, ss)
except:
log.error("work through %s error: %s", ww,
utils.current_exception_info())
print 'module names of ', top_dir, ' is ', module_dir_names[top_dir]
def reload_class_methods(self, current_meths, new_meths):
try:
if not does_local_ip_matched():
return
# in current_funcs, but not in new_funcs, need to restore profile
diff_restore = list(set(current_meths).difference(set(new_meths)))
for aa in diff_restore:
arr = str(aa).split(',')
patt = build_pattern(arr[0], arr[1], arr[2])
if patt in self.method_wrappers:
mw = self.method_wrappers[patt]
mw.restore_method_aspect()
self.method_wrappers.pop(patt)
if aa in new_meths:
new_meths.remove(aa)
# not in current_funcs, but in new_funcs, need to add profile
# diff_profile = list(set(new_meths).difference(set(current_meths)))
for aa in new_meths:
arr = str(aa).split(',')
self.profile_class_method(arr[0], arr[1], arr[2])
except:
log.info("reload_class_methods %s , %s error: %s ",
str(current_meths), str(new_meths),
utils.current_exception_info())
def __wrap_method_internal(self):
for k, v in self.mod.__dict__.items():
try:
if not inspect.isclass(v) or getattr(
v, '__module__') != self.mod.__name__:
continue
meta_dict = self.get_aspect_clazz_methods(k, v)
if len(meta_dict) == 0:
continue
meta_clazz = v
for kk, meth in meta_clazz.__dict__.items():
if kk.startswith("__"):
continue
# meth = getattr(meta_clazz, kk)
meth_func = getattr(meta_clazz, kk)
# print '11 ', kk, ', \t ', type(meth), ' \t, ', type(meth_func)
if not is_function(meth) and not is_method(meth):
continue
func_name = kk
if func_name not in meta_dict:
continue
if str(type(meth)).find('staticmethod') > -1:
# setattr(meta_clazz, kk, profile_aspect()(meth_func, meta_clazz))
setattr(meta_clazz, kk,
staticmethod(profile_aspect()(meth_func)))
elif str(type(meth)).find('function') > -1:
setattr(meta_clazz, kk, profile_aspect()(meth))
elif str(type(meth)).find('classmethod') > -1:
# setattr(meta_clazz, kk, profile_aspect()(getattr(meth_func, 'im_func'), meta_clazz))
setattr(
meta_clazz, kk,
classmethod(profile_aspect()(getattr(
meth_func, 'im_func'))))
else:
setattr(meta_clazz, kk, profile_aspect()(meth_func))
log.info("code profile for method - %s.%s.%s",
meta_clazz.__module__, meta_clazz, func_name)
setattr(self.mod, k, meta_clazz)
self.aspect_clazzs[v] = meta_dict
self.clazz_origin[self.mod.__name__ + "." + k] = v
except:
log.error('first aspect module class %s error: %s',
self.mod.__name__, utils.current_exception_info())
def profile_module_function(self, module_name, function_pattern):
try:
if not does_local_ip_matched():
return
patt = build_pattern(module_name, function_pattern)
if patt in self.function_wrappers:
fw = self.function_wrappers[patt]
fw.restore_function_aspect()
self.function_wrappers.pop(patt)
fw = FunctionWrapper(module_name, function_pattern)
fw.wrap_function()
self.function_wrappers[patt] = fw
except:
log.error("profile_module_function error: %s",
utils.current_exception_info())
def __wrap_function_internal(self, k, v):
try:
if k.startswith("__"):
return
if inspect.isfunction(v) and getattr(
v, '__module__') == self.mod.__name__:
function_full_name = self.mod.__name__ + "." + v.func_name
if self.does_match_function_aspect(
function_full_name) and not is_inner_function(v):
setattr(self.mod, k, profile_aspect()(v))
self.module_func_origin[function_full_name] = v
log.info("code profile for function - %s",
function_full_name)
except:
log.error('first aspect module %s error: %s', self.mod.__name__,
utils.current_exception_info())
def profile_class_method(self, module_name, clazz_name, method_pattern):
try:
if not does_local_ip_matched():
return
patt = build_pattern(module_name, clazz_name, method_pattern)
if patt in self.method_wrappers:
mw = self.method_wrappers[patt]
mw.restore_method_aspect()
self.method_wrappers.pop(patt)
mw = MethodWrapper(module_name, clazz_name, method_pattern)
mw.wrap_method()
self.method_wrappers[patt] = mw
except:
log.error("profile_class_method error: %s",
utils.current_exception_info())
def __init__(self, module_name, function_pattern):
self.module_name = module_name
self.function_pattern = function_pattern
self.mod = get_module(module_name)
try:
if self.mod is None:
import importlib
importlib.import_module(module_name)
self.mod = get_module(module_name)
except:
log.error('try to import %s error: %s', module_name,
utils.current_exception_info())
if self.mod is None:
raise Exception("module_name %s do not exist ", module_name)
self.function_patterns.add(build_pattern(module_name,
function_pattern))
def __search_all_module(self, top_dir):
if top_dir is None or top_dir == '' or not does_profile_switch_on(
) or top_dir in self.searched_dirs:
return
reload_num = 0
if top_dir.endswith('.reload'):
reload_num = 1
top_dir = top_dir[0:-7]
if top_dir.endswith('.unload'):
reload_num = 2
top_dir = top_dir[0:-7]
try:
log.info("os.getcwd() is %s, top_dir is %s, ip is %s", os.getcwd(),
top_dir, utils.local_ip)
curr_real_path = os.path.realpath(__file__)
work_dir = curr_real_path[0:curr_real_path.find('/core/profile')]
log.info("work_dir is %s, os.path.dirname(__file__) is %s, %s",
work_dir, os.path.dirname(__file__), curr_real_path)
self.__find_sub_modules(work_dir, top_dir)
self.searched_dirs.add(top_dir)
log.info("all_module_names size is %s", len(all_module_names))
if reload_num == 1:
self.__reload_all()
if reload_num == 2:
self.__unload_all()
except:
log.info("search_all_module %s error: %s", top_dir,
utils.current_exception_info())
def __wrap_other_function(self, mo):
try:
mo_name = mo.__name__
self_mod_name = self.mod.__name__
if mo_name == self_mod_name:
return
for kk, vv in mo.__dict__.items():
if kk.startswith("__"): # or not inspect.ismodule(vv)
continue
full_func_name = "%s.%s" % (mo_name, kk)
if inspect.ismodule(
vv) and vv is self.mod: # inspect.ismodule(vv) and
self.module_func_origin[full_func_name] = vv
setattr(mo, kk, self.mod)
self.add_wrapped_func('__mod_match__', full_func_name, vv)
log.info("other code profile for import as %s at %s", kk,
mo_name)
elif inspect.isfunction(vv):
vv_mo_name = vv.__module__
matched_patt = self.does_match_function_aspect(vv_mo_name +
'.' + kk)
if not matched_patt:
continue
# self.module_func_origin[full_func_name] = vv
setattr(mo, kk, profile_aspect()(vv))
self.add_wrapped_func(matched_patt, full_func_name, vv)
log.info("other code profile for from import %s at %s",
full_func_name, mo_name)
except:
all_module_names.remove(mo.__name__)
log.warn("%s error: %s", mo.__name__,
utils.current_exception_info())
def wrap_method(self):
if not does_profile_switch_on() or not does_local_ip_matched():
log.info(
'code_profile_switch is False or does_local_ip_matched is False'
)
return
log.info('start profile method for %s.%s pattern is %s',
self.mod.__name__, self.clazz_name, self.method_pattern)
self.__wrap_method_internal()
# aspect functions that are imported classes and objects by from .. import
imported_modules = []
for na in all_module_names.copy():
if na == self.mod.__name__ or get_module(na) is None:
continue
mo = get_module(na)
imported_modules.append(na)
# log.info("search imported clazz %s", na)
for kk, vv in mo.__dict__.items():
try:
if kk.startswith("__") or inspect.ismodule(vv):
continue
if inspect.isclass(vv) and vv in self.aspect_clazzs:
self.__wrap_other_clazz_method(kk, vv, mo)
else:
self.__wrap_other_object_method(kk, vv)
except:
all_module_names.remove(na)
log.warn("%s error: %s", na,
utils.current_exception_info())
log.info("search imported module %s", str(imported_modules))
log.info('end profile method for %s.%s pattern is %s',
self.mod.__name__, self.clazz_name, self.method_pattern)